nokolexbor 0.2.0

data/vendor/lexbor/source/lexbor/utils/warc.h

@@ -0,0 +1,126 @@
+/*
+ * Copyright (C) 2019 Alexander Borisov
+ *
+ * Author: Alexander Borisov <borisov@lexbor.com>
+ */
+
+#ifndef LXB_UTILS_WARC_H
+#define LXB_UTILS_WARC_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "lexbor/utils/base.h"
+
+#include "lexbor/core/mraw.h"
+#include "lexbor/core/str.h"
+#include "lexbor/core/array_obj.h"
+
+
+typedef struct lxb_utils_warc lxb_utils_warc_t;
+
+typedef struct {
+    lexbor_str_t type;
+    double       number;
+}
+lxb_utils_warc_version_t;
+
+typedef struct {
+    lexbor_str_t name;
+    lexbor_str_t value;
+}
+lxb_utils_warc_field_t;
+
+typedef lxb_status_t
+(*lxb_utils_warc_header_cb_f)(lxb_utils_warc_t *warc);
+
+typedef lxb_status_t
+(*lxb_utils_warc_content_cb_f)(lxb_utils_warc_t *warc, const lxb_char_t *data,
+                               const lxb_char_t *end);
+typedef lxb_status_t
+(*lxb_utils_warc_content_end_cb_f)(lxb_utils_warc_t *warc);
+
+struct lxb_utils_warc {
+    lexbor_mraw_t                   *mraw;
+    lexbor_array_obj_t              *fields;
+
+    lexbor_str_t                    tmp;
+    lxb_utils_warc_version_t        version;
+
+    lxb_utils_warc_header_cb_f      header_cb;
+    lxb_utils_warc_content_cb_f     content_cb;
+    lxb_utils_warc_content_end_cb_f content_end_cb;
+    void                            *ctx;
+
+    size_t                          content_length;
+    size_t                          content_read;
+    size_t                          count;
+
+    const char                      *error;
+    unsigned                        state;
+    unsigned                        ends;
+    bool                            skip;
+};
+
+
+LXB_API lxb_utils_warc_t *
+lxb_utils_warc_create(void);
+
+LXB_API lxb_status_t
+lxb_utils_warc_init(lxb_utils_warc_t *warc, lxb_utils_warc_header_cb_f h_cd,
+                    lxb_utils_warc_content_cb_f c_cb,
+                    lxb_utils_warc_content_end_cb_f c_end_cb, void *ctx);
+
+LXB_API lxb_status_t
+lxb_utils_warc_clear(lxb_utils_warc_t *warc);
+
+LXB_API lxb_utils_warc_t *
+lxb_utils_warc_destroy(lxb_utils_warc_t *warc, bool self_destroy);
+
+
+LXB_API lxb_status_t
+lxb_utils_warc_parse_file(lxb_utils_warc_t *warc, FILE *fh);
+
+/*
+ * We must call lxb_warc_parse_eof after processing.
+ * Before new processing we must call lxb_warc_clear
+ * if previously ends with error.
+ */
+LXB_API lxb_status_t
+lxb_utils_warc_parse(lxb_utils_warc_t *warc,
+                     const lxb_char_t **data, const lxb_char_t *end);
+
+LXB_API lxb_status_t
+lxb_utils_warc_parse_eof(lxb_utils_warc_t *warc);
+
+
+LXB_API lxb_utils_warc_field_t *
+lxb_utils_warc_header_field(lxb_utils_warc_t *warc, const lxb_char_t *name,
+                            size_t len, size_t offset);
+
+LXB_API lxb_status_t
+lxb_utils_warc_header_serialize(lxb_utils_warc_t *warc, lexbor_str_t *str);
+
+/*
+ * Inline functions
+ */
+lxb_inline size_t
+lxb_utils_warc_content_length(lxb_utils_warc_t *warc)
+{
+    return warc->content_length;
+}
+
+/*
+ * No inline functions for ABI.
+ */
+LXB_API size_t
+lxb_utils_warc_content_length_noi(lxb_utils_warc_t *warc);
+
+
+#ifdef __cplusplus
+} /* extern "C" */
+#endif
+
+#endif /* LXB_UTILS_WARC_H */
+

data/vendor/lexbor/utils/lexbor/css/selectors/pseudo.py

@@ -0,0 +1,231 @@
+
+import sys, re, os
+
+# Find and append run script run dir to module search path
+ABS_PATH = os.path.dirname(os.path.abspath(__file__))
+sys.path.append("{}/../../lexbor/".format(ABS_PATH))
+
+import LXB
+
+pseudo_default_state_success = "lxb_css_selectors_state_success"
+
+pseudo_classes = [
+    # https://drafts.csswg.org/selectors-4/#overview
+    "warning", "any-link", "link", "visited", "local-link", "target",
+    "target-within", "scope", "current", "past", "future", "active",
+    "hover", "focus", "focus-within", "focus-visible", "enabled", 
+    "disabled", "read-write", "read-only", "placeholder-shown",
+    "default", "checked", "indeterminate", "valid", "invalid",
+    "in-range", "out-of-range", "required", "optional", "blank",
+    "user-invalid", "root", "empty", "first-child", "last-child",
+    "only-child", "first-of-type", "last-of-type", "only-of-type",
+
+    # https://fullscreen.spec.whatwg.org/#index
+    "fullscreen"
+]
+
+pseudo_class_functions_dict = {
+    # [callback end function, can empty]
+    #
+    # https://drafts.csswg.org/selectors-4/#overview
+    "not": [pseudo_default_state_success, False, "CLOSE"],
+    "is": ["lxb_css_selectors_state_forgiving", False, "CLOSE"],
+    "where": ["lxb_css_selectors_state_forgiving", False, "CLOSE"],
+    "has": ["lxb_css_selectors_state_forgiving_relative", False, "DESCENDANT"],
+    "dir": [pseudo_default_state_success, False, "CLOSE"],
+    "lang": [pseudo_default_state_success, False, "CLOSE"],
+    "current": [pseudo_default_state_success, False, "CLOSE"],
+    "nth-child": [pseudo_default_state_success, False, "CLOSE"],
+    "nth-last-child": [pseudo_default_state_success, False, "CLOSE"],
+    "nth-of-type": [pseudo_default_state_success, False, "CLOSE"],
+    "nth-last-of-type": [pseudo_default_state_success, False, "CLOSE"],
+    "nth-col": [pseudo_default_state_success, False, "CLOSE"],
+    "nth-last-col": [pseudo_default_state_success, False, "CLOSE"]
+}
+
+pseudo_class_functions = list(pseudo_class_functions_dict.keys())
+
+pseudo_elements = [
+    # https://drafts.csswg.org/css-pseudo-4/#index
+    "after", "before", "first-letter", "first-line", "grammar-error",
+    "inactive-selection", "marker", "placeholder", "selection",
+    "spelling-error", "target-text",
+
+    # https://fullscreen.spec.whatwg.org/#index
+    "backdrop"
+]
+
+pseudo_element_functions_dict = {
+}
+
+pseudo_element_functions = list(pseudo_element_functions_dict.keys())
+
+pseudo_classes.sort()
+pseudo_classes.insert(0, "_undef")
+
+pseudo_class_functions.sort()
+pseudo_class_functions.insert(0, "_undef")
+
+pseudo_elements.sort()
+pseudo_elements.insert(0, "_undef")
+
+pseudo_element_functions.sort()
+pseudo_element_functions.insert(0, "_undef")
+
+class Pseudo:
+    prefix = "LXB_CSS_SELECTOR_"
+
+    def make(self, pseude_name, arr, names = None):
+        idx = 0
+        state = ""
+        entries = []
+        type_data_name = ""
+        state_declaration = []
+        state_bodies = []
+        data_name = "lxb_css_selectors_pseudo_data_{}".format(pseude_name)
+
+        make_func = names != None
+
+        if make_func == True:
+            type_data_name = "lxb_css_selectors_pseudo_data_func_t"
+        else:
+            type_data_name = "lxb_css_selectors_pseudo_data_t"
+
+        res = LXB.Res(type_data_name, data_name, False,
+                      self.make_enum_name(pseude_name, "_LAST_ENTRY"))
+
+        frmt_enum = LXB.FormatEnum("{}{}_id_t".format(self.prefix.lower(),
+                                                      self.make_name(pseude_name)))
+
+        for name in arr:
+            enum_name = self.make_enum_name(pseude_name, name)
+
+            frmt_enum.append(enum_name, "0x{0:04x}".format(idx))
+            idx += 1
+
+            if make_func == True:
+                state = "lxb_css_selectors_state_{}_{}".format(self.make_name(pseude_name),
+                                                               self.make_name(name))
+                state_declaration.append(self.make_state_declaration(state))
+                state_bodies.append(self.make_state_body(state))
+
+                success = pseudo_default_state_success
+                can_empty = False
+                combinator = "CLOSE"
+
+                if name in names:
+                    success = names[name][0]
+                    can_empty = names[name][1]
+                    combinator = names[name][2]
+
+                if name == "_undef":
+                    res.append("{{(lxb_char_t *) \"#undef\", 6, {},\n     {}, {},"
+                               " {},\n     {}}}".format(enum_name,
+                                        success, state, str(can_empty).lower(),
+                                        self.make_combinator_name(combinator)))
+                    continue
+                else:
+                    res.append("{{(lxb_char_t *) \"{}\", {}, {},\n     {}, {},"
+                               " {},\n     {}}}".format(name, len(name),
+                                        enum_name, success, state, str(can_empty).lower(),
+                                        self.make_combinator_name(combinator)))
+            else:
+                if name == "_undef":
+                    res.append("{{(lxb_char_t *) \"#undef\", 6, {}}}".format(enum_name))
+                    continue
+                else:
+                    res.append("{{(lxb_char_t *) \"{}\", {}, {}}}".format(name, len(name),
+                                                                          enum_name))
+
+            entries.append({"key": name,
+                            "value": "(void *) &{}[{}]".format(data_name,  enum_name)})
+
+        frmt_enum.append(self.make_enum_name(pseude_name, "_LAST_ENTRY"),
+                                             "0x{0:04x}".format(idx))
+
+        shs = LXB.SHS(entries, 0, False)
+        test = shs.make_test(5, 256)
+        shs.table_size_set(test[0][2])
+
+        result_enum = frmt_enum.build()
+        result_res = res.create(rate = 1, is_const = True)
+        result_sh = shs.create("lxb_css_selectors_{}_shs".format(pseude_name), rate = 1)
+
+        print(self.shs_stat(shs))
+
+        return [result_enum, result_res, result_sh, state_declaration, state_bodies]
+
+    def make_combinator_name(self, name):
+        name = re.sub(r"[^a-zA-Z0-9_]", "_", name)
+        return "{}COMBINATOR_{}".format(self.prefix, name.upper())
+
+    def make_enum_name(self, pseudo_name, name):
+        name = re.sub(r"[^a-zA-Z0-9_]", "_", name)
+        pseudo_name = re.sub(r"[^a-zA-Z0-9_]", "_", pseudo_name)
+        return "{}{}_{}".format(self.prefix, pseudo_name.upper(), name.upper())
+
+    def make_name(self, name):
+        return re.sub("[^a-zA-Z0-9]", "_", name)
+
+    def shs_stat(self, shs):
+       return "Max deep {}; Used {} of {}".format(shs.max, shs.used, shs.table_size)
+
+    def make_state_declaration(self, name):
+        return ("bool\n{}(lxb_css_parser_t *parser,\n"
+                "    lxb_css_syntax_token_t *token, void *ctx)").format(name)
+
+    def make_state_body(self, name):
+        return "{}\n{{\n    return true;\n}}".format(self.make_state_declaration(name))
+
+    def save(self, const_temp_file, const_save_to, res_temp_file, res_save_to):
+        res_class = ps.make("pseudo_class", pseudo_classes)
+        res_class_function = ps.make("pseudo_class_function", pseudo_class_functions, pseudo_class_functions_dict)
+        res_element = ps.make("pseudo_element", pseudo_elements)
+        res_element_function = ps.make("pseudo_element_function", pseudo_element_functions, pseudo_element_functions_dict)
+
+        # const
+        lxb_temp = LXB.Temp(const_temp_file, const_save_to)
+
+        lxb_temp.pattern_append("%%BODY%%",
+                                "{}\n\n{}\n\n{}\n\n{}".format('\n'.join(res_class[0]),
+                                                        '\n'.join(res_class_function[0]),
+                                                        '\n'.join(res_element[0]),
+                                                        '\n'.join(res_element_function[0])))
+
+        lxb_temp.build()
+        lxb_temp.save()
+
+        print("Const saved to {}".format(const_save_to))
+
+        # res
+        lxb_temp = LXB.Temp(res_temp_file, res_save_to)
+
+        lxb_temp.pattern_append("%%SHS_DATA%%",
+                                "{}\n\n{}\n\n{}\n\n{}".format(''.join(res_class[1]),
+                                                        ''.join(res_class_function[1]),
+                                                        ''.join(res_element[1]),
+                                                        ''.join(res_element_function[1])))
+        lxb_temp.pattern_append("%%SHS%%",
+                                "{}\n\n{}\n\n{}\n\n{}".format(''.join(res_class[2]),
+                                                        ''.join(res_class_function[2]),
+                                                        ''.join(res_element[2]),
+                                                        ''.join(res_element_function[2])))
+
+        lxb_temp.build()
+        lxb_temp.save()
+
+        print("Res saved to {}".format(res_save_to))
+        print("Done")
+
+        yn = input("Print declarations? (y - yes, n - no): ")
+        if yn.lower() == 'y':
+            print('LXB_API ', ';\n\nLXB_API '.join(res_class_function[3]), ';', sep='')
+            print('LXB_API ', ';\n\nLXB_API '.join(res_element_function[3]), ';', sep='')
+            print('\n\n'.join(res_class_function[4]))
+            print('\n\n'.join(res_element_function[4]))
+
+if __name__ == "__main__":
+    ps = Pseudo()
+
+    ps.save("tmp/const.h", "../../../../source/lexbor/css/selectors/pseudo_const.h",
+            "tmp/res.h", "../../../../source/lexbor/css/selectors/pseudo_res.h")

data/vendor/lexbor/utils/lexbor/css/selectors/tmp/const.h

@@ -0,0 +1,21 @@
+/*
+ * Copyright (C) 2020-2021 Alexander Borisov
+ *
+ * Author: Alexander Borisov <borisov@lexbor.com>
+ */
+
+/*
+ * Caution!
+ * This file generated by the script "utils/lexbor/css/selectors/pseudo.py"!
+ * Do not change this file!
+ */
+
+
+#ifndef LXB_CSS_SELECTOS_PSEUDO_CONST_H
+#define LXB_CSS_SELECTOS_PSEUDO_CONST_H
+
+
+%%BODY%%
+
+
+#endif /* LXB_CSS_SELECTOS_PSEUDO_CONST_H */

data/vendor/lexbor/utils/lexbor/css/selectors/tmp/res.h

@@ -0,0 +1,26 @@
+/*
+ * Copyright (C) 2020-2021 Alexander Borisov
+ *
+ * Author: Alexander Borisov <borisov@lexbor.com>
+ */
+
+/*
+ * Caution!
+ * This file generated by the script "utils/lexbor/css/selectors/pseudo.py"!
+ * Do not change this file!
+ */
+
+
+#ifndef LXB_CSS_SELECTOS_PSEUDO_RES_H
+#define LXB_CSS_SELECTOS_PSEUDO_RES_H
+
+#include "lexbor/core/shs.h"
+#include "lexbor/css/selectors/pseudo_const.h"
+
+
+%%SHS_DATA%%
+
+%%SHS%%
+
+
+#endif /* LXB_CSS_SELECTOS_PSEUDO_RES_H */

data/vendor/lexbor/utils/lexbor/css/syntax/definitions.py

@@ -0,0 +1,49 @@
+import os, sys
+
+ABS_PATH = os.path.dirname(os.path.abspath(__file__))
+sys.path.append("{}/../../".format(ABS_PATH))
+
+name_start_code_point = "0x01"
+name_code_point = "0x02"
+
+import lexbor.LXB
+
+def_before = "#define LXB_CSS_SYNTAX_RES_NAME_START " + name_start_code_point
+
+def name():
+    res = lexbor.LXB.Res("lxb_char_t", "lxb_css_syntax_res_name_map", True, 256)
+
+    for code in range(0, 128):
+        # U+0061 LATIN SMALL LETTER A (a) and U+007A LATIN SMALL LETTER Z (z)
+        if code >= 0x61 and code <= 0x7A:
+            res.append(name_start_code_point)
+
+        # U+0041 LATIN CAPITAL LETTER A (A) and U+005A LATIN CAPITAL LETTER Z (Z)
+        elif code >= 0x41 and code <= 0x5A:
+            res.append(name_start_code_point)
+
+        # U+0030 DIGIT ZERO (0) and U+0039 DIGIT NINE (9)
+        elif code >= 0x30 and code <= 0x39:
+            res.append(name_code_point)
+
+        # U+005F LOW LINE (_)
+        elif code == 0x5F:
+            res.append(name_start_code_point)
+
+        # U+002D HYPHEN-MINUS (-)
+        elif code == 0x2D:
+            res.append(name_code_point)
+
+        # all other
+        else:
+            res.append("0x00")
+
+    for code in range(128, 255):
+        res.append(name_start_code_point)
+
+    map_list = res.create(10, True, def_before)
+
+    return ''.join(map_list)
+
+if __name__ == "__main__":
+    print(name())

data/vendor/lexbor/utils/lexbor/css/syntax/token_res.py

@@ -0,0 +1,54 @@
+
+import os, sys
+
+# Find and append run script run dir to module search path
+ABS_PATH = os.path.dirname(os.path.abspath(__file__))
+sys.path.append("{}/../../lexbor/".format(ABS_PATH))
+
+import LXB
+
+enum_tokens = {
+    "undefined": "LXB_CSS_SYNTAX_TOKEN_UNDEF",
+    "ident": "LXB_CSS_SYNTAX_TOKEN_IDENT",
+    "function": "LXB_CSS_SYNTAX_TOKEN_FUNCTION",
+    "at-keyword": "LXB_CSS_SYNTAX_TOKEN_AT_KEYWORD",
+    "hash": "LXB_CSS_SYNTAX_TOKEN_HASH",
+    "string": "LXB_CSS_SYNTAX_TOKEN_STRING",
+    "bad-string": "LXB_CSS_SYNTAX_TOKEN_BAD_STRING",
+    "url": "LXB_CSS_SYNTAX_TOKEN_URL",
+    "bad-url": "LXB_CSS_SYNTAX_TOKEN_BAD_URL",
+    "delim": "LXB_CSS_SYNTAX_TOKEN_DELIM",
+    "number": "LXB_CSS_SYNTAX_TOKEN_NUMBER",
+    "percentage": "LXB_CSS_SYNTAX_TOKEN_PERCENTAGE",
+    "dimension": "LXB_CSS_SYNTAX_TOKEN_DIMENSION",
+    "whitespace": "LXB_CSS_SYNTAX_TOKEN_WHITESPACE",
+    "CDO": "LXB_CSS_SYNTAX_TOKEN_CDO",
+    "CDC": "LXB_CSS_SYNTAX_TOKEN_CDC",
+    "colon": "LXB_CSS_SYNTAX_TOKEN_COLON",
+    "semicolon": "LXB_CSS_SYNTAX_TOKEN_SEMICOLON",
+    "comma": "LXB_CSS_SYNTAX_TOKEN_COMMA",
+    "left-square-bracket": "LXB_CSS_SYNTAX_TOKEN_LS_BRACKET",
+    "right-square-bracket": "LXB_CSS_SYNTAX_TOKEN_RS_BRACKET",
+    "left-parenthesis": "LXB_CSS_SYNTAX_TOKEN_L_PARENTHESIS",
+    "right-parenthesis": "LXB_CSS_SYNTAX_TOKEN_R_PARENTHESIS",
+    "left-curly-bracket": "LXB_CSS_SYNTAX_TOKEN_LC_BRACKET",
+    "right-curly-bracket": "LXB_CSS_SYNTAX_TOKEN_RC_BRACKET",
+    "comment": "LXB_CSS_SYNTAX_TOKEN_COMMENT",
+    "end-of-file": "LXB_CSS_SYNTAX_TOKEN__EOF"
+}
+
+def create(name, enum_tokens):
+    tlist = []
+
+    for (key, value) in enum_tokens.items():
+        tlist.append({'key': key.lower(), 'value': "(void *) " + value})
+
+    shs = LXB.SHS(tlist, 0, True)
+
+    test = shs.make_test(5, 128)
+    shs.table_size_set(test[0][2])
+
+    return shs.create(name)
+
+if __name__ == "__main__":
+    print("".join(create("lxb_css_syntax_token_res_name_shs_map", enum_tokens)))

data/vendor/lexbor/utils/lexbor/css/syntax/tokenizer_code_map.py

@@ -0,0 +1,36 @@
+import os, sys
+
+ABS_PATH = os.path.dirname(os.path.abspath(__file__))
+sys.path.append("{}/../../".format(ABS_PATH))
+
+import lexbor.res
+
+def make_ascii_map():
+    entries = []
+    ascii_map = lexbor.res.ascii_names_map
+
+    for code in range(0, 128):
+        if ascii_map[code][1]:
+            entries.append("NULL, /* 0x%s; '%s'; %s */" 
+                % (format(code, '02X'), *ascii_map[code]))
+        else:
+            entries.append("NULL, /* 0x%s; '%s' */" 
+                % (format(code, '02X'), ascii_map[code][0]))
+
+    for code in range(128, 255):
+        entries.append("NULL, /* 0x%s */" % format(code, 'X'))
+
+    entries.append("NULL  /* 0x%s */" % format(code + 1, 'X'))
+
+    map_list = []
+
+    map_list.append("static const lxb_css_syntax_tokenizer_state_f")
+    map_list.append("lxb_css_syntax_state_res_map[256] =")
+    map_list.append("{")
+    map_list.append('    ' + '\n    '.join(entries))
+    map_list.append("};")
+
+    return '\n'.join(map_list)
+
+if __name__ == "__main__":
+    print(make_ascii_map())

data/vendor/lexbor/version

@@ -0,0 +1 @@
+LEXBOR_VERSION=2.1.0