cfacter 3.11.0.rc.20180314

data/ext/facter/cpp-hocon/lib/src/config_parse_options.cc

@@ -0,0 +1,101 @@
+#include <hocon/config_parse_options.hpp>
+#include <hocon/config_includer.hpp>
+#include <leatherman/locale/locale.hpp>
+
+// Mark string for translation (alias for leatherman::locale::format)
+using leatherman::locale::_;
+
+using namespace std;
+
+namespace hocon {
+
+    config_parse_options::config_parse_options(shared_string origin_desc,
+            bool allow_missing, shared_includer includer, config_syntax syntax) :
+        _syntax(syntax), _origin_description(move(origin_desc)),
+        _allow_missing(allow_missing), _includer(move(includer)) {}
+
+    config_parse_options::config_parse_options(): config_parse_options(nullptr, true, nullptr, config_syntax::CONF) {}
+
+    config_parse_options config_parse_options::defaults() {
+        return config_parse_options{nullptr, true, nullptr, config_syntax::UNSPECIFIED };
+    }
+
+    config_parse_options config_parse_options::set_syntax(config_syntax syntax) const
+    {
+        return config_parse_options{_origin_description, _allow_missing, _includer, syntax};
+    }
+
+    config_syntax const& config_parse_options::get_syntax() const
+    {
+        return _syntax;
+    }
+
+    config_parse_options config_parse_options::set_origin_description(shared_string origin_description) const
+    {
+        return config_parse_options{move(origin_description), _allow_missing, _includer, _syntax};
+    }
+
+
+    shared_string const& config_parse_options::get_origin_description() const
+    {
+        return _origin_description;
+    }
+
+    config_parse_options config_parse_options::with_fallback_origin_description(shared_string origin_description) const
+    {
+        if (!_origin_description) {
+            return set_origin_description(origin_description);
+        } else {
+            return *this;
+        }
+    }
+
+    config_parse_options config_parse_options::set_allow_missing(bool allow_missing) const
+    {
+        return config_parse_options{_origin_description, allow_missing, _includer, _syntax};
+    }
+
+    bool config_parse_options::get_allow_missing() const
+    {
+        return _allow_missing;
+    }
+
+    config_parse_options config_parse_options::set_includer(shared_includer includer) const
+    {
+        return config_parse_options{ _origin_description, _allow_missing, move(includer), _syntax};
+    }
+
+    config_parse_options config_parse_options::prepend_includer(shared_includer includer) const
+    {
+        if (!includer) {
+            throw runtime_error(_("null includer passed to prepend_includer"));
+        }
+        if (_includer == includer) {
+            return *this;
+        } else if (_includer) {
+            return set_includer(includer->with_fallback(_includer));
+        } else {
+            return set_includer(includer);
+        }
+    }
+
+    config_parse_options config_parse_options::append_includer(shared_includer includer) const
+    {
+        if (!includer) {
+            throw runtime_error(_("null includer passed to append_includer"));
+        }
+        if (_includer == includer) {
+            return *this;
+        } else if (_includer) {
+            return set_includer(_includer->with_fallback(move(includer)));
+        } else {
+            return set_includer(includer);
+        }
+    }
+
+    shared_includer const& config_parse_options::get_includer() const
+    {
+        return _includer;
+    }
+
+}  // namespace hocon

data/ext/facter/cpp-hocon/lib/src/config_parser.cc

@@ -0,0 +1,399 @@
+#include <internal/config_parser.hpp>
+#include <hocon/config_exception.hpp>
+#include <hocon/config_object.hpp>
+#include <internal/tokens.hpp>
+#include <internal/simple_includer.hpp>
+#include <internal/substitution_expression.hpp>
+#include <internal/nodes/config_node_comment.hpp>
+#include <internal/nodes/config_node_complex_value.hpp>
+#include <internal/nodes/config_node_simple_value.hpp>
+#include <internal/nodes/config_node_object.hpp>
+#include <internal/nodes/config_node_array.hpp>
+#include <internal/nodes/config_node_concatenation.hpp>
+#include <internal/nodes/config_node_field.hpp>
+#include <internal/nodes/config_node_include.hpp>
+#include <internal/values/config_concatenation.hpp>
+#include <internal/values/config_reference.hpp>
+#include <internal/values/simple_config_object.hpp>
+#include <internal/values/simple_config_list.hpp>
+#include <leatherman/locale/locale.hpp>
+
+// Mark string for translation (alias for leatherman::locale::format)
+using leatherman::locale::_;
+
+namespace hocon { namespace config_parser {
+    using namespace std;
+
+    shared_value parse(shared_ptr<const config_node_root> document,
+            shared_origin origin,
+            config_parse_options options,
+            shared_include_context include_context)
+    {
+        parse_context context {options.get_syntax(), origin, document,
+                               simple_includer::make_full(options.get_includer()), include_context};
+        return context.parse();
+    }
+
+    parse_context::parse_context(config_syntax flavor, shared_origin origin, shared_ptr<const config_node_root> document,
+            shared_ptr<const full_includer> includer, shared_include_context include_context) :
+        _line_number(1), _document(document), _includer(includer), _include_context(include_context),
+        _flavor(flavor), _base_origin(origin), array_count(0)
+    {}
+
+    shared_origin parse_context::line_origin() const {
+        return _base_origin->with_line_number(_line_number);
+    }
+
+    path parse_context::full_current_path() const {
+        // pathStack has top of stack at front
+        if (_path_stack.empty()) {
+            throw bug_or_broken_exception(_("Bug in parser; tried to get current path when at root"));
+        } else {
+            return path {_path_stack.front()};
+        }
+    }
+
+    shared_value parse_context::parse_value(shared_node_value n, vector<string>& comments)
+    {
+        auto starting_array_count = array_count;
+
+        shared_value v = [&]() -> shared_value {
+            if (auto val = dynamic_pointer_cast<const config_node_simple_value>(n)) {
+                return val->get_value();
+            } else if (auto val = dynamic_pointer_cast<const config_node_object>(n)) {
+                return parse_object(val);
+            } else if (auto val = dynamic_pointer_cast<const config_node_array>(n)) {
+                return parse_array(val);
+            } else if (auto val = dynamic_pointer_cast<const config_node_concatenation>(n)) {
+                return parse_concatenation(val);
+            } else {
+                auto &deref_n = *n;
+                throw parse_exception(*line_origin(),
+                                      _("Expecting a value but got wrong node type: {1}", typeid(deref_n).name()));
+            }
+        }();
+
+        if (!comments.empty()) {
+            auto old_origin = dynamic_pointer_cast<const simple_config_origin>(v->origin());
+            if (!old_origin) {
+                throw bug_or_broken_exception(_("origin should be a simple_config_origin"));
+            }
+            v = v->with_origin(old_origin->prepend_comments(move(comments)));
+        }
+
+        if (array_count != starting_array_count) {
+            throw bug_or_broken_exception(_("Bug in config parser: unbalanced array count"));
+        }
+        return v;
+    }
+
+    shared_object parse_context::create_value_under_path(path p, shared_value value) {
+        // for path foo.bar, we are creating { "foo" : { "bar" : value } }
+        vector<shared_string> keys;
+
+        shared_string key = p.first();
+        path remaining = p.remainder();
+        while (key != nullptr) {
+            keys.push_back(key);
+            if (remaining.empty()) {
+                break;
+            } else {
+                key = remaining.first();
+                remaining = remaining.remainder();
+            }
+        }
+
+        auto current = keys.end();
+        current--;
+        auto new_value = unordered_map<string, shared_value>({ {**current, value} });
+        shared_object obj = make_shared<simple_config_object>(value->origin()->with_comments(vector<string>{}),
+                                                              new_value);
+
+        while (current != keys.begin()) {
+            current--;
+            new_value = unordered_map<string, shared_value>({ {**current, obj} });
+            obj = make_shared<simple_config_object>(value->origin()->with_comments(vector<string>{}), new_value);
+        }
+
+        return obj;
+    }
+
+    void parse_context::parse_include(unordered_map<string, shared_value> & values,
+                                      shared_ptr<const config_node_include> n) {
+        shared_object obj;
+        switch (n->kind()) {
+            case config_include_kind::FILE:
+                obj = dynamic_pointer_cast<const config_object>(_includer->include_file(_include_context, n->name()));
+                break;
+            case config_include_kind::CLASSPATH:
+                // TODO: implement include_resource (?)
+                throw config_exception(_("full_includer::include_resource not implemented"));
+                break;
+            case config_include_kind::HEURISTIC:
+                obj = dynamic_pointer_cast<const config_object>(_includer->include(_include_context, n->name()));
+                break;
+            default:
+                throw config_exception(_("should not be reached"));
+                break;
+        }
+
+        // we really should make this work, but for now throwing an
+        // exception is better than producing an incorrect result.
+        // See https://github.com/typesafehub/config/issues/160
+        if (array_count > 0 && obj->get_resolve_status() != resolve_status::RESOLVED) {
+            throw config_exception(leatherman::locale::translate("Due to current limitations of the config parser, when an include statement is nested inside a list value,\n${} substitutions inside the included file cannot be resolved correctly. Either move the include outside of the list value or\nremove the ${} statements from the included file."));
+        }
+
+        if (!_path_stack.empty()) {
+            auto prefix = full_current_path();
+            obj = dynamic_pointer_cast<const config_object>(obj->relativized(prefix.to_string()));
+        }
+
+        for (auto &pair : *obj) {
+            auto &key = pair.first;
+            auto &v = pair.second;
+            auto iter = values.find(key);
+            if (iter != values.end()) {
+                auto existing = values[key];
+                values[key] = dynamic_pointer_cast<const config_value>(v->with_fallback(existing));
+            } else {
+                values.emplace(key, v);
+            }
+        }
+    }
+
+    shared_object parse_context::parse_object(shared_node_object n)
+    {
+        unordered_map<string, shared_value> values;
+        auto object_origin = line_origin();
+        bool last_was_newline = false;
+
+        auto nodes = n->children();
+        vector<string> comments;
+        for (size_t i = 0; i < nodes.size(); ++i) {
+            auto node = nodes.at(i);
+            if (auto comment = dynamic_pointer_cast<const config_node_comment>(node)) {
+                last_was_newline = false;
+                comments.push_back(comment->comment_text());
+            } else if (auto singletoken = dynamic_pointer_cast<const config_node_single_token>(node)) {
+                if (tokens::is_newline(singletoken->get_token())) {
+                    _line_number++;
+                    if (last_was_newline) {
+                        // Drop all comments if there was a blank line and start a new comment block
+                        comments.clear();
+                    }
+                    last_was_newline = true;
+                }
+            } else if (auto include = dynamic_pointer_cast<const config_node_include>(node)) {
+                if (_flavor != config_syntax::JSON) {
+                    parse_include(values, include);
+                    last_was_newline = false;
+                }
+            } else if (auto field = dynamic_pointer_cast<const config_node_field>(node)) {
+                last_was_newline = false;
+                auto path = field->path()->get_path();
+                auto field_comments = field->comments();
+                comments.insert(comments.end(), field_comments.begin(), field_comments.end());
+
+                // path must be on-stack while we parse the value
+                _path_stack.push_back(path);
+                if (field->separator() == tokens::plus_equals_token()) {
+                    // we really should make this work, but for now throwing
+                    // an exception is better than producing an incorrect
+                    // result. See
+                    // https://github.com/typesafehub/config/issues/160
+                    if (array_count > 0) {
+                        throw parse_exception(*line_origin(), leatherman::locale::translate("Due to current limitations of the config parser, += does not work nested inside a list. += expands to a ${} substitution and the path in ${} cannot currently refer to list elements. You might be able to move the += outside of the list and then refer to it from inside the list with ${}."));
+                    }
+
+                    // because we will put it in an array after the fact so
+                    // we want this to be incremented during the parseValue
+                    // below in order to throw the above exception.
+                    array_count += 1;
+                }
+
+                auto value_node = field->get_value();
+                // comments from the key token go to the value token
+                auto new_value = parse_value(value_node, comments);
+
+                if (field->separator() == tokens::plus_equals_token()) {
+                    array_count -= 1;
+
+                    vector<shared_value> concat;
+                    concat.reserve(2);
+                    auto previous_ref = make_shared<config_reference>(new_value->origin(), make_shared<substitution_expression>(full_current_path(), true));
+                    auto list = make_shared<simple_config_list>(new_value->origin(), vector<shared_value>({new_value}));
+                    concat.push_back(previous_ref);
+                    concat.push_back(list);
+                    new_value = config_concatenation::concatenate(concat);
+                }
+
+                // Grab any trailing comments on the same line
+                if (i < nodes.size() - 1) {
+                    ++i;
+                    while (i < nodes.size()) {
+                        if (auto comment = dynamic_pointer_cast<const config_node_comment>(nodes.at(i))) {
+                            auto old_origin = dynamic_pointer_cast<const simple_config_origin>(new_value->origin());
+                            if (!old_origin) {
+                                throw bug_or_broken_exception(_("expected origin to be simple_config_origin"));
+                            }
+                            new_value = new_value->with_origin(old_origin->append_comments({comment->comment_text()}));
+                            break;
+                        } else if (auto curr = dynamic_pointer_cast<const config_node_single_token>(nodes.at(i))) {
+                            if (curr->get_token() == tokens::comma_token() ||
+                                tokens::is_ignored_whitespace(curr->get_token())) {
+                                // keep searching, as there could still be a comment
+                            } else {
+                                --i;
+                                break;
+                            }
+                        } else {
+                            --i;
+                            break;
+                        }
+                        ++i;
+                    }
+                }
+
+                _path_stack.pop_back();
+
+                auto key = path.first();
+                auto remaining = path.remainder();
+
+                if (remaining.empty()) {
+                    auto existing = values.find(*key);
+                    if (existing != values.end()) {
+                        // In strict JSON, dups should be an error; while in
+                        // our custom config language, they should be merged
+                        // if the value is an object (or substitution that
+                        // could become an object).
+
+                        if (_flavor == config_syntax::JSON) {
+                            throw parse_exception(*line_origin(), _("JSON does not allow duplicate fields: '{1}' was already seen at {2}", *key, existing->second->origin()->description()));
+                        } else {
+                            new_value = dynamic_pointer_cast<const config_value>(new_value->with_fallback(existing->second));
+                            assert(new_value);
+                        }
+                    }
+                    values[*key] = new_value;
+                } else {
+                    if (_flavor == config_syntax::JSON) {
+                        throw new bug_or_broken_exception(_("somehow got multi-element path in JSON mode"));
+                    }
+
+                    shared_object obj = create_value_under_path(remaining, new_value);
+                    auto existing = values.find(*key);
+                    if (existing != values.end()) {
+                        obj = dynamic_pointer_cast<const config_object>(obj->with_fallback(existing->second));
+                        assert(obj);
+                    }
+                    values[*key] = obj;
+                }
+            }
+        }
+
+        return make_shared<simple_config_object>(object_origin, move(values));
+    }
+
+    static shared_ptr<const simple_config_origin> as_origin(shared_origin o) {
+        auto simple_o = dynamic_pointer_cast<const simple_config_origin>(o);
+        if (!simple_o) {
+            throw bug_or_broken_exception(_("origin was not a simple_config_origin"));
+        }
+        return simple_o;
+    }
+
+    shared_value parse_context::parse_array(shared_node_array n) {
+        ++array_count;
+
+        auto array_origin = line_origin();
+        auto values = vector<shared_value>();
+
+        bool last_was_new_line = false;
+        auto comments = vector<string>();
+
+        shared_value v;
+
+        for (auto node : n->children()) {
+            if (auto comment = dynamic_pointer_cast<const config_node_comment>(node)) {
+                comments.push_back(comment->comment_text());
+                last_was_new_line = false;
+            } else if (auto singletoken = dynamic_pointer_cast<const config_node_single_token>(node)) {
+                if (tokens::is_newline(singletoken->get_token())) {
+                    _line_number++;
+                    if (last_was_new_line && v == nullptr) {
+                        comments.clear();
+                    } else if (v) {
+                        values.push_back(v->with_origin(as_origin(v->origin())->append_comments(move(comments))));
+                        comments.clear();
+                        v = nullptr;
+                    }
+                    last_was_new_line = true;
+                }
+            } else if (auto value = dynamic_pointer_cast<const abstract_config_node_value>(node)) {
+                last_was_new_line = false;
+                if (v) {
+                    values.push_back(v->with_origin(as_origin(v->origin())->append_comments(move(comments))));
+                    comments.clear();
+                }
+                v = parse_value(value, comments);
+            }
+        }
+
+        // There shouldn't be any comments at this point, but add them just in case
+        if (v) {
+            values.push_back(v->with_origin(as_origin(v->origin())->append_comments(move(comments))));
+        }
+        --array_count;
+        return make_shared<simple_config_list>(move(array_origin), move(values));
+    }
+
+    shared_value parse_context::parse_concatenation(shared_node_concatenation n) {
+        if (_flavor == config_syntax::JSON) {
+            throw bug_or_broken_exception(_("Found a concatenation node in JSON"));
+        }
+
+        vector<shared_value> values;
+        for (auto& node : n->children()) {
+            if (auto value_node = dynamic_pointer_cast<const abstract_config_node_value>(node)) {
+                vector<string> comments;
+                values.push_back(parse_value(value_node, comments));
+            }
+        }
+
+        return config_concatenation::concatenate(move(values));
+    }
+
+    shared_value parse_context::parse()
+    {
+        shared_value result;
+        vector<string> comments;
+        bool last_was_newline = false;
+
+        for (auto&& node : _document->children()) {
+            if (auto ptr = dynamic_pointer_cast<const config_node_comment>(node)) {
+                comments.push_back(ptr->comment_text());
+                last_was_newline = false;
+            } else if (auto ptr = dynamic_pointer_cast<const config_node_single_token>(node)) {
+                auto t = ptr->get_token();
+                if (t->get_token_type() == token_type::NEWLINE) {
+                    ++_line_number;
+                    if (last_was_newline && !result) {
+                        comments.clear();
+                    } else if (result) {
+                        auto origin = dynamic_pointer_cast<const simple_config_origin>(result->origin());
+                        assert(origin);
+                        result = result->with_origin(origin->append_comments(move(comments)));
+                        break;
+                    }
+                    last_was_newline = true;
+                }
+            } else if (auto ptr = dynamic_pointer_cast<const config_node_complex_value>(node)) {
+                result = parse_value(ptr, comments);
+                last_was_newline = false;
+            }
+        }
+
+        return result;
+    }
+}}  // namespace hocon::config_parser