cfacter 3.11.0.rc.20180314

data/ext/facter/cpp-hocon/lib/tests/test_utils.hpp

@@ -0,0 +1,139 @@
+#pragma once
+
+#include <internal/simple_config_origin.hpp>
+#include <internal/substitution_expression.hpp>
+#include <internal/tokens.hpp>
+#include <internal/values/config_string.hpp>
+#include <internal/values/config_boolean.hpp>
+#include <internal/values/config_double.hpp>
+#include <internal/values/config_long.hpp>
+#include <internal/values/config_int.hpp>
+#include <internal/values/config_null.hpp>
+#include <internal/values/config_reference.hpp>
+#include <internal/values/config_concatenation.hpp>
+#include <internal/nodes/config_node_simple_value.hpp>
+
+#include <string>
+#include <memory>
+#include <vector>
+#include <hocon/path.hpp>
+#include <internal/nodes/config_node_field.hpp>
+#include <internal/nodes/config_node_single_token.hpp>
+#include <internal/nodes/config_node_comment.hpp>
+
+#define REQUIRE_STRING_CONTAINS(str, match) REQUIRE(std::string(str).find(match) != std::string::npos)
+#define REQUIRE_STRING_NOT_CONTAINS(str, match) REQUIRE(std::string(str).find(match) == std::string::npos)
+
+namespace hocon { namespace test_utils {
+
+    shared_origin fake_origin(std::string description = "fake", int line_number = 0);
+
+    /** Tokens */
+    std::shared_ptr<value> string_token(std::string text, config_string_type type = config_string_type::QUOTED);
+
+    std::shared_ptr<value> bool_token(bool boolean);
+
+    std::shared_ptr<value> double_token(double number, std::string original_text);
+
+    std::shared_ptr<value> int_token(int number, std::string original_text);
+
+    std::shared_ptr<value> long_token(int64_t number, std::string original_text);
+
+    std::shared_ptr<value> null_token();
+
+    std::shared_ptr<substitution> substitution_token(shared_token inner, bool optional = false);
+
+    std::shared_ptr<line> line_token(int line_number);
+
+    std::shared_ptr<ignored_whitespace> whitespace_token(std::string whitespace);
+
+    std::shared_ptr<unquoted_text> unquoted_text_token(std::string text);
+
+    std::shared_ptr<double_slash_comment> double_slash_comment_token(std::string text);
+
+    std::shared_ptr<hash_comment> hash_comment_token(std::string text);
+
+    /** Nodes */
+    std::shared_ptr<config_node_single_token> colon_node();
+
+    std::shared_ptr<config_node_single_token> open_brace_node();
+
+    std::shared_ptr<config_node_single_token> close_brace_node();
+
+    std::shared_ptr<config_node_single_token> space_node();
+
+    std::shared_ptr<config_node_single_token> comma_node();
+
+    std::shared_ptr<config_node_field> node_key_value_pair(std::shared_ptr<config_node_path> key,
+                                                           shared_node_value value);
+
+    std::shared_ptr<config_node_path> node_key(std::string key);
+
+    std::shared_ptr<config_node_simple_value> int_node(int number);
+
+    std::shared_ptr<config_node_simple_value> long_node(int64_t number);
+
+    std::shared_ptr<config_node_simple_value> double_node(double number);
+
+    std::shared_ptr<config_node_simple_value> string_node(std::string text);
+
+    std::shared_ptr<config_node_simple_value> null_node();
+
+    std::shared_ptr<config_node_simple_value> bool_node(bool value);
+
+    std::shared_ptr<config_node_simple_value> unquoted_text_node(std::string text);
+
+    std::shared_ptr<config_node_simple_value> substitution_node(shared_token key, bool optional);
+
+    std::shared_ptr<config_node_single_token> line_node(int line_number);
+
+    std::shared_ptr<config_node_single_token> whitespace_node(std::string whitespace);
+
+    std::shared_ptr<config_node_comment> double_slash_comment_node(std::string text);
+
+    // it's important that these do NOT use the public API to create the
+    // instances, because we may be testing that the public API returns the
+    // right instance by comparing to these, so using public API here would
+    // make the test compare public API to itself.
+    std::shared_ptr<config_int> int_value(int i);
+    // TODO: remaining value helpers
+
+    std::shared_ptr<config_boolean> bool_value(bool b);
+
+    std::shared_ptr<config_null> null_value();
+
+    std::shared_ptr<config_string> string_value(std::string s);
+
+    std::shared_ptr<config_double> double_value(double d);
+
+    std::shared_ptr<config_reference> subst(std::string ref, bool optional = false);
+
+    std::shared_ptr<config_concatenation> subst_in_string(std::string ref, bool optional = false);
+
+    /** Paths */
+    path test_path(std::initializer_list<std::string> path_elements);
+
+    shared_object parse_object(std::string);
+
+    shared_config parse_config(std::string);
+
+    struct parse_test {
+        parse_test(std::string t, bool lbe = false, bool wm = false)
+            : test(move(t)), lift_behavior_unexpected(lbe), whitespace_matters(wm) { }
+
+        std::string test;
+        bool lift_behavior_unexpected, whitespace_matters;
+    };
+
+    std::vector<parse_test> const& invalid_json();
+
+    std::vector<parse_test> const& invalid_conf();
+
+    std::vector<parse_test> const& valid_json();
+
+    std::vector<parse_test> const& valid_conf();
+
+    std::vector<parse_test> whitespace_variations(std::vector<parse_test> const& tests, bool valid_in_lift);
+
+    std::string fixture_path(std::string fixture_name);
+}}  // namespace hocon::test_utils

data/ext/facter/cpp-hocon/lib/tests/token_test.cc

@@ -0,0 +1,108 @@
+#include <catch.hpp>
+
+#include <internal/token.hpp>
+#include <internal/tokens.hpp>
+#include <internal/simple_config_origin.hpp>
+#include <internal/values/config_boolean.hpp>
+#include <internal/values/config_boolean.hpp>
+#include "test_utils.hpp"
+
+using namespace std;
+using namespace hocon;
+using namespace hocon::test_utils;
+
+TEST_CASE("token equality", "[tokens]") {
+
+    SECTION("singleton token equality") {
+        REQUIRE(*tokens::start_token() == *tokens::start_token());
+        REQUIRE_FALSE(*tokens::start_token() == *tokens::end_token());
+    }
+
+    SECTION("value token equality") {
+        value true_value(make_shared<config_boolean>(fake_origin("fake"), true));
+        value other_true(make_shared<config_boolean>(fake_origin("other fake"), true));
+        value false_value(make_shared<config_boolean>(fake_origin("fake"), false));
+
+        REQUIRE(true_value == other_true);
+        REQUIRE_FALSE(true_value == false_value);
+    }
+
+    SECTION("line token equality") {
+        line line_value(fake_origin());
+        line other_line(fake_origin("other fake"));
+        ignored_whitespace not_a_line(fake_origin(), "   ");
+
+        REQUIRE(line_value == other_line);
+        REQUIRE_FALSE(line_value == not_a_line);
+    }
+
+    SECTION("unquoted text token equality") {
+        unquoted_text text(fake_origin(), "no quotes");
+        unquoted_text other_text(fake_origin("other fake"), "no quotes");
+        unquoted_text different_text(fake_origin(), "still no quotes");
+        ignored_whitespace not_unquoted_text(fake_origin(), "   ");
+
+        REQUIRE(text == other_text);
+        REQUIRE_FALSE(text == different_text);
+        REQUIRE_FALSE(text == not_unquoted_text);
+    }
+
+    SECTION("ignored whitespace equality") {
+        ignored_whitespace three_spaces(fake_origin(), "   ");
+        ignored_whitespace three_more_spaces(fake_origin("other fake"), "   ");
+        ignored_whitespace two_spaces(fake_origin(), "  ");
+        unquoted_text not_whitespace(fake_origin(), "foo");
+
+        REQUIRE(three_spaces == three_more_spaces);
+        REQUIRE_FALSE(three_spaces == two_spaces);
+        REQUIRE_FALSE(three_spaces == not_whitespace);
+    }
+
+    SECTION("problem token equality") {
+        problem problem_token(fake_origin(), "test exception", "it broke", false);
+        problem other_problem(fake_origin("other fake"), "test exception", "it broke", false);
+        ignored_whitespace not_a_problem(fake_origin(), "   ");
+
+        REQUIRE(problem_token == other_problem);
+        REQUIRE_FALSE(problem_token == not_a_problem);
+    }
+
+    SECTION("comment equality") {
+        comment comment_token(fake_origin(), "my comment");
+        comment other_comment(fake_origin("other fake"), "my comment");
+        comment different_comment(fake_origin(), "a different comment");
+        ignored_whitespace not_a_comment(fake_origin(), "   ");
+
+        REQUIRE(comment_token == other_comment);
+        REQUIRE_FALSE(comment_token == different_comment);
+        REQUIRE_FALSE(comment_token == not_a_comment);
+
+        // Comment comparison only deals with the actual text, not the marker
+        hash_comment hash(fake_origin(), "my comment");
+        double_slash_comment slashes(fake_origin("other origin"), "my comment");
+
+        REQUIRE(hash == slashes);
+    }
+
+    SECTION("substitution equality") {
+        token_list expression1;
+        expression1.push_back(make_shared<line>(fake_origin()));
+        expression1.push_back(make_shared<line>(fake_origin("other")));
+
+        token_list expression1_dup;
+        expression1_dup.push_back(make_shared<line>(fake_origin()));
+        expression1_dup.push_back(make_shared<line>(fake_origin("other")));
+
+        token_list expression2;
+        expression2.push_back(make_shared<line>(fake_origin()));
+
+        substitution sub(fake_origin(), false, move(expression1));
+        substitution other_sub(fake_origin("other"), false, move(expression1_dup));
+        substitution different_sub(fake_origin(), false, move(expression2));
+        line not_a_sub(fake_origin("line"));
+
+        REQUIRE(sub == other_sub);
+        REQUIRE_FALSE(sub == different_sub);
+        REQUIRE_FALSE(sub == not_a_sub);
+    }
+}

data/ext/facter/cpp-hocon/lib/tests/tokenizer_test.cc

@@ -0,0 +1,694 @@
+#include <catch.hpp>
+#include <internal/tokenizer.hpp>
+#include "test_utils.hpp"
+#include <iostream>
+
+using namespace std;
+using namespace hocon;
+using namespace hocon::test_utils;
+
+token_list tokenize_as_list(string const& source) {
+    token_iterator iter(fake_origin(), unique_ptr<istringstream>(new istringstream(source)), true);
+    // get all the tokens from the string and put them in a vector
+    token_list tokens;
+    while (iter.has_next()) {
+        tokens.push_back(iter.next());
+    }
+    return tokens;
+}
+
+void tokenizer_test(string source, token_list expected) {
+    token_list result = tokenize_as_list(source);
+    for (size_t i = 0; i < expected.size(); i++) {
+        if (!(*expected[i] == *result[i])) {
+            // Debugging info
+            cerr << result[i]->to_string() << " but expected " << expected[i]->to_string() << " -- ";
+            cerr << "i = " + std::to_string(i);
+        }
+        REQUIRE(*expected[i] == *result[i]);
+    }
+}
+
+TEST_CASE("tokenize basic strings", "[tokenizer]") {
+    SECTION("tokenize empty string") {
+        string source = "";
+        token_list expected {
+                tokens::start_token(),
+                tokens::end_token()
+        };
+        tokenizer_test(source, expected);
+    }
+
+    SECTION("tokenize newlines") {
+        string source = "\n\n";
+        token_list expected {
+                tokens::start_token(),
+                make_shared<line>(fake_origin()->with_line_number(1)),
+                make_shared<line>(fake_origin()->with_line_number(2)),
+                tokens::end_token()
+        };
+        tokenizer_test(source, expected);
+    }
+}
+
+TEST_CASE("tokenize all types", "[tokenizer]") {
+    SECTION("tokenize all types with no spaces") {
+        string source = ",:=}{][+=\"foo\"\"\"\"bar\"\"\"true3.14false42null${a.b}${?x.y}${\"c.d\"}\n";
+        token_list expected {
+                tokens::start_token(),
+                tokens::comma_token(),
+                tokens::colon_token(),
+                tokens::equals_token(),
+                tokens::close_curly_token(),
+                tokens::open_curly_token(),
+                tokens::close_square_token(),
+                tokens::open_square_token(),
+                tokens::plus_equals_token(),
+                string_token("foo"),
+                string_token("bar"),
+                bool_token(true),
+                double_token(3.14, "3.14"),
+                bool_token(false),
+                int_token(42, "42"),
+                null_token(),
+                substitution_token(unquoted_text_token("a.b"), false),
+                substitution_token(unquoted_text_token("x.y"), true),
+                substitution_token(string_token("\"c.d\""), false),
+                line_token(1),
+                tokens::end_token()
+        };
+        tokenizer_test(source, expected);
+    }
+
+    SECTION("tokenize all types with spaces") {
+        string source = " , : = } { ] [ += \"foo\" \"\"\"bar\"\"\" 42 true 3.14 false null ${a.b} ${?x.y} ${\"c.d\"} \n ";
+        token_list expected {
+                tokens::start_token(),
+                whitespace_token(" "),
+                tokens::comma_token(),
+                whitespace_token(" "),
+                tokens::colon_token(),
+                whitespace_token(" "),
+                tokens::equals_token(),
+                whitespace_token(" "),
+                tokens::close_curly_token(),
+                whitespace_token(" "),
+                tokens::open_curly_token(),
+                whitespace_token(" "),
+                tokens::close_square_token(),
+                whitespace_token(" "),
+                tokens::open_square_token(),
+                whitespace_token(" "),
+                tokens::plus_equals_token(),
+                whitespace_token(" "),
+                string_token("foo"),
+                unquoted_text_token(" "),
+                string_token("bar"),
+                unquoted_text_token(" "),
+                int_token(42, "42"),
+                unquoted_text_token(" "),
+                bool_token(true),
+                unquoted_text_token(" "),
+                double_token(3.14, "3.14"),
+                unquoted_text_token(" "),
+                bool_token(false),
+                unquoted_text_token(" "),
+                null_token(),
+                unquoted_text_token(" "),
+                substitution_token(unquoted_text_token("a.b"), false),
+                unquoted_text_token(" "),
+                substitution_token(unquoted_text_token("x.y"), true),
+                unquoted_text_token(" "),
+                substitution_token(string_token("\"c.d\""), false),
+                whitespace_token(" "),
+                line_token(1),
+                whitespace_token(" "),
+                tokens::end_token()
+        };
+        tokenizer_test(source, expected);
+    }
+
+    SECTION("tokenize all types with multiple spaces") {
+        string source = "   ,   :   =   }   {   ]   [   +=   \"foo\"   \"\"\"bar\"\"\"   42   true   3.14   false   null   ${a.b}   ${?x.y}   ${\"c.d\"}  \n   ";
+        token_list expected {
+                tokens::start_token(),
+                whitespace_token("   "),
+                tokens::comma_token(),
+                whitespace_token("   "),
+                tokens::colon_token(),
+                whitespace_token("   "),
+                tokens::equals_token(),
+                whitespace_token("   "),
+                tokens::close_curly_token(),
+                whitespace_token("   "),
+                tokens::open_curly_token(),
+                whitespace_token("   "),
+                tokens::close_square_token(),
+                whitespace_token("   "),
+                tokens::open_square_token(),
+                whitespace_token("   "),
+                tokens::plus_equals_token(),
+                whitespace_token("   "),
+                string_token("foo"),
+                unquoted_text_token("   "),
+                string_token("bar"),
+                unquoted_text_token("   "),
+                int_token(42, "42"),
+                unquoted_text_token("   "),
+                bool_token(true),
+                unquoted_text_token("   "),
+                double_token(3.14, "3.14"),
+                unquoted_text_token("   "),
+                bool_token(false),
+                unquoted_text_token("   "),
+                null_token(),
+                unquoted_text_token("   "),
+                substitution_token(unquoted_text_token("a.b"), false),
+                unquoted_text_token("   "),
+                substitution_token(unquoted_text_token("x.y"), true),
+                unquoted_text_token("   "),
+                substitution_token(string_token("\"c.d\""), false),
+                whitespace_token("  "),
+                line_token(1),
+                whitespace_token("   "),
+                tokens::end_token()
+        };
+        tokenizer_test(source, expected);
+    }
+}
+
+TEST_CASE("unquoted text and booleans", "[tokenizer]") {
+    SECTION("true and unquoted text") {
+        string source = "truefoo";
+        token_list expected {
+                tokens::start_token(),
+                bool_token(true),
+                unquoted_text_token("foo"),
+                tokens::end_token()
+        };
+        tokenizer_test(source, expected);
+    }
+
+    SECTION("false and unquoted text") {
+        string source = "falsefoo";
+        token_list expected {
+                tokens::start_token(),
+                bool_token(false),
+                unquoted_text_token("foo"),
+                tokens::end_token()
+        };
+        tokenizer_test(source, expected);
+    }
+
+    SECTION("null and unquoted text") {
+        string source = "nullfoo";
+        token_list expected {
+                tokens::start_token(),
+                null_token(),
+                unquoted_text_token("foo"),
+                tokens::end_token()
+        };
+        tokenizer_test(source, expected);
+    }
+
+    SECTION("unquoted text containing true") {
+        string source = "footrue";
+        token_list expected {
+                tokens::start_token(),
+                unquoted_text_token("footrue"),
+                tokens::end_token()
+        };
+        tokenizer_test(source, expected);
+    }
+
+    SECTION("unquoted text containing space true") {
+        string source = "foo true";
+        token_list expected {
+                tokens::start_token(),
+                unquoted_text_token("foo"),
+                unquoted_text_token(" "),
+                bool_token(true),
+                tokens::end_token()
+        };
+        tokenizer_test(source, expected);
+    }
+
+    SECTION("true and space and unquoted text") {
+        string source = "true foo";
+        token_list expected {
+                tokens::start_token(),
+                bool_token(true),
+                unquoted_text_token(" "),
+                unquoted_text_token("foo"),
+                tokens::end_token()
+        };
+        tokenizer_test(source, expected);
+    }
+}
+
+TEST_CASE("unquoted strings with special cases", "[tokenizer]") {
+    SECTION("unquoted text containing slash") {
+        string source = "a/b/c";
+        token_list expected {
+                tokens::start_token(),
+                unquoted_text_token("a/b/c"),
+                tokens::end_token()
+        };
+        tokenizer_test(source, expected);
+
+        source = "/";
+        expected = {
+                tokens::start_token(),
+                unquoted_text_token("/"),
+                tokens::end_token()
+        };
+        tokenizer_test(source, expected);
+
+        source = "/ /";
+        expected = {
+                tokens::start_token(),
+                unquoted_text_token("/"),
+                unquoted_text_token(" "),
+                unquoted_text_token("/"),
+                tokens::end_token()
+        };
+        tokenizer_test(source, expected);
+    }
+
+    SECTION("unquoted text discards external spaces") {
+        string source = "   foo   \n";
+        token_list expected {
+                tokens::start_token(),
+                whitespace_token("   "),
+                unquoted_text_token("foo"),
+                whitespace_token("   "),
+                line_token(1),
+                tokens::end_token()
+        };
+        tokenizer_test(source, expected);
+    }
+
+    SECTION("unquoted text keeps internal spaces") {
+        string source = "    foo  bar baz   \n";
+        token_list expected {
+                tokens::start_token(),
+                whitespace_token("    "),
+                unquoted_text_token("foo"),
+                unquoted_text_token("  "),
+                unquoted_text_token("bar"),
+                unquoted_text_token(" "),
+                unquoted_text_token("baz"),
+                whitespace_token("   "),
+                line_token(1),
+                tokens::end_token()
+        };
+        tokenizer_test(source, expected);
+    }
+
+    SECTION("mix quoted and unquoted") {
+        string source = "   foo\"bar\"baz   \n";
+        token_list expected {
+                tokens::start_token(),
+                whitespace_token("   "),
+                unquoted_text_token("foo"),
+                string_token("bar"),
+                unquoted_text_token("baz"),
+                whitespace_token("   "),
+                line_token(1),
+                tokens::end_token()
+        };
+        tokenizer_test(source, expected);
+    }
+}
+
+TEST_CASE("escape sequence", "[tokenizer]") {
+    SECTION("unicode infinity symbol") {
+        string source = "\"\\u221E\"";
+        token_list expected {
+                tokens::start_token(),
+                string_token(u8"\u221E"),
+                tokens::end_token()
+        };
+        tokenizer_test(source, expected);
+    }
+
+    SECTION("null byte") {
+        string source = " \"\\u0000\" ";
+        token_list expected {
+                tokens::start_token(),
+                whitespace_token(" "),
+                string_token(""),
+                whitespace_token(" "),
+                tokens::end_token()
+        };
+        tokenizer_test(source, expected);
+    }
+
+    SECTION("various escape codes") {
+        string source = " \"\\\"\\\\/\\b\\f\\n\\r\\t\" ";
+        token_list expected {
+                tokens::start_token(),
+                whitespace_token(" "),
+                string_token("\"\\/\b\f\n\r\t"),
+                whitespace_token(" "),
+                tokens::end_token()
+        };
+        tokenizer_test(source, expected);
+    }
+
+    SECTION("unicode F") {
+        string source = " \"\\u0046\" ";
+        token_list expected {
+                tokens::start_token(),
+                whitespace_token(" "),
+                string_token("F"),
+                whitespace_token(" "),
+                tokens::end_token()
+        };
+        tokenizer_test(source, expected);
+    }
+
+    SECTION("two unicode F's") {
+        string source = " \"\\u0046\\u0046\" ";
+        token_list expected {
+                tokens::start_token(),
+                whitespace_token(" "),
+                string_token("FF"),
+                whitespace_token(" "),
+                tokens::end_token()
+        };
+        tokenizer_test(source, expected);
+    }
+}
+
+TEST_CASE("triple quoted strings") {
+    SECTION("trivial triple quoted string") {
+        string source = "\"\"\"bar\"\"\"";
+        token_list expected {
+                tokens::start_token(),
+                string_token("bar"),
+                tokens::end_token()
+        };
+        tokenizer_test(source, expected);
+    }
+
+    SECTION("trailing quotes in triple quoted string") {
+        string source = "\"\"\"\"\"\"\"\"";
+        token_list expected {
+                tokens::start_token(),
+                string_token("\"\""),
+                tokens::end_token()
+        };
+        tokenizer_test(source, expected);
+    }
+
+    SECTION("no esacpe in triple quoted strings") {
+        string source = "\"\"\"\\n\"\"\"";
+        token_list expected {
+                tokens::start_token(),
+                string_token("\\n"),
+                tokens::end_token()
+        };
+        tokenizer_test(source, expected);
+    }
+
+    SECTION("new line in triple quoted string") {
+        string source = "\"\"\"foo\nbar\"\"\"";
+        token_list expected {
+                tokens::start_token(),
+                string_token("foo\nbar"),
+                tokens::end_token()
+        };
+        tokenizer_test(source, expected);
+    }
+}
+
+TEST_CASE("comments", "[tokenizer]") {
+    SECTION("double slash comment") {
+        string source = "//";
+        token_list expected {
+                tokens::start_token(),
+                double_slash_comment_token(""),
+                tokens::end_token()
+        };
+        tokenizer_test(source, expected);
+    }
+
+    SECTION("hash comment") {
+        string source = "#";
+        token_list expected {
+                tokens::start_token(),
+                hash_comment_token(""),
+                tokens::end_token()
+        };
+        tokenizer_test(source, expected);
+    }
+
+    SECTION("two slashes in quoted string is string") {
+        string source = "\"//bar\"";
+        token_list expected {
+                tokens::start_token(),
+                string_token("//bar"),
+                tokens::end_token()
+        };
+        tokenizer_test(source, expected);
+    }
+
+    SECTION("hash in quoted string is string") {
+        string source = "\"#bar\"";
+        token_list expected {
+                tokens::start_token(),
+                string_token("#bar"),
+                tokens::end_token()
+        };
+        tokenizer_test(source, expected);
+    }
+
+    SECTION("slash comment after unquoted text") {
+        string source = "bar//comment";
+        token_list expected {
+                tokens::start_token(),
+                unquoted_text_token("bar"),
+                double_slash_comment_token("comment"),
+                tokens::end_token()
+        };
+        tokenizer_test(source, expected);
+    }
+
+    SECTION("hash comment after unquoted text") {
+        string source = "bar#comment";
+        token_list expected {
+                tokens::start_token(),
+                unquoted_text_token("bar"),
+                hash_comment_token("comment"),
+                tokens::end_token()
+        };
+        tokenizer_test(source, expected);
+    }
+
+    SECTION("slash comment after int") {
+        string source = "10//comment";
+        token_list expected {
+                tokens::start_token(),
+                int_token(10, "10"),
+                double_slash_comment_token("comment"),
+                tokens::end_token()
+        };
+        tokenizer_test(source, expected);
+    }
+
+    SECTION("hash comment after int") {
+        string source = "10#comment";
+        token_list expected {
+                tokens::start_token(),
+                int_token(10, "10"),
+                hash_comment_token("comment"),
+                tokens::end_token()
+        };
+        tokenizer_test(source, expected);
+    }
+
+    SECTION("slash comment with newline") {
+        string source = "10//comment\n12";
+        token_list expected {
+                tokens::start_token(),
+                int_token(10, "10"),
+                double_slash_comment_token("comment"),
+                line_token(1),
+                int_token(12, "12"),
+                tokens::end_token()
+        };
+        tokenizer_test(source, expected);
+    }
+
+    SECTION("hash comment with newline") {
+        string source = "10#comment\n12";
+        token_list expected {
+                tokens::start_token(),
+                int_token(10, "10"),
+                hash_comment_token("comment"),
+                line_token(1),
+                int_token(12, "12"),
+                tokens::end_token()
+        };
+        tokenizer_test(source, expected);
+    }
+
+    SECTION("slash comment on multiples lines with whitespace") {
+        string source = "   //comment\r\n   //comment2   \n//comment3   \n\n//comment4";
+        token_list expected {
+                tokens::start_token(),
+                whitespace_token("   "),
+                double_slash_comment_token("comment\r"),
+                line_token(1),
+                whitespace_token("   "),
+                double_slash_comment_token("comment2   "),
+                line_token(2),
+                double_slash_comment_token("comment3   "),
+                line_token(3),
+                line_token(4),
+                double_slash_comment_token("comment4"),
+                tokens::end_token()
+        };
+        tokenizer_test(source, expected);
+    }
+
+    SECTION("hash comment on multiples lines with whitespace") {
+        string source = "   #comment\r\n   #comment2   \n#comment3   \n\n#comment4";
+        token_list expected {
+                tokens::start_token(),
+                whitespace_token("   "),
+                hash_comment_token("comment\r"),
+                line_token(1),
+                whitespace_token("   "),
+                hash_comment_token("comment2   "),
+                line_token(2),
+                hash_comment_token("comment3   "),
+                line_token(3),
+                line_token(4),
+                hash_comment_token("comment4"),
+                tokens::end_token()
+        };
+        tokenizer_test(source, expected);
+    }
+}
+
+TEST_CASE("brackets and braces", "[tokenizer]") {
+    SECTION("open curlies") {
+        string source = "{{";
+        token_list expected{
+                tokens::start_token(),
+                tokens::open_curly_token(),
+                tokens::open_curly_token(),
+                tokens::end_token()
+        };
+        tokenizer_test(source, expected);
+    }
+
+    SECTION("close curlies") {
+        string source = "}}";
+        token_list expected{
+                tokens::start_token(),
+                tokens::close_curly_token(),
+                tokens::close_curly_token(),
+                tokens::end_token()
+        };
+        tokenizer_test(source, expected);
+    }
+
+    SECTION("open and close curlies") {
+        string source = "{}";
+        token_list expected{
+                tokens::start_token(),
+                tokens::open_curly_token(),
+                tokens::close_curly_token(),
+                tokens::end_token()
+        };
+        tokenizer_test(source, expected);
+    }
+
+    SECTION("open squares") {
+        string source = "[[";
+        token_list expected{
+                tokens::start_token(),
+                tokens::open_square_token(),
+                tokens::open_square_token(),
+                tokens::end_token()
+        };
+        tokenizer_test(source, expected);
+    }
+
+    SECTION("close curlies") {
+        string source = "]]";
+        token_list expected{
+                tokens::start_token(),
+                tokens::close_square_token(),
+                tokens::close_square_token(),
+                tokens::end_token()
+        };
+        tokenizer_test(source, expected);
+    }
+
+    SECTION("open and close curlies") {
+        string source = "[]";
+        token_list expected{
+                tokens::start_token(),
+                tokens::open_square_token(),
+                tokens::close_square_token(),
+                tokens::end_token()
+        };
+        tokenizer_test(source, expected);
+    }
+}
+
+void test_for_config_error(string source) {
+    token_iterator iter(fake_origin(), unique_ptr<istringstream>(new istringstream(source)), true);
+    while (iter.has_next()) {
+        iter.next();
+    }
+}
+
+TEST_CASE("catch syntax erros", "[tokenizer]") {
+    SECTION("nothing after backslash") {
+        string source = " \"\\\" ";
+        REQUIRE_THROWS(test_for_config_error(source));
+    }
+
+    SECTION("\\q is not a valid escape sequence") {
+        string source = " \"\\q\" ";
+        REQUIRE_THROWS(test_for_config_error(source));
+    }
+
+    SECTION("unicode byte sequence missing bytes") {
+        string source = " \"\\u012\" ";
+        REQUIRE_THROWS(test_for_config_error(source));
+
+        source = " \"\\u01\" ";
+        REQUIRE_THROWS(test_for_config_error(source));
+
+        source = " \"\\u1\" ";
+        REQUIRE_THROWS(test_for_config_error(source));
+
+        source = " \"\\u\" ";
+        REQUIRE_THROWS(test_for_config_error(source));
+    }
+
+    SECTION("missing closing quotes") {
+        string source = "\"";
+        REQUIRE_THROWS(test_for_config_error(source));
+
+        source = "\"abc";
+        REQUIRE_THROWS(test_for_config_error(source));
+    }
+
+    SECTION("invalid lone characters") {
+        string source = "\"\\\"";
+        REQUIRE_THROWS(test_for_config_error(source));
+
+        source = "$";
+        REQUIRE_THROWS(test_for_config_error(source));
+
+        source = "${";
+        REQUIRE_THROWS(test_for_config_error(source));
+    }
+}