lite-email-parser 2.0.1 → 2.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/binding.gyp +19 -16
- package/core/include/html_extractor.h +100 -0
- package/core/include/html_parser.h +13 -0
- package/core/src/html_extractor.cpp +662 -0
- package/core/src/html_parser.cpp +221 -0
- package/deps/gumbo-parser/src/attribute.c +44 -0
- package/deps/gumbo-parser/src/attribute.h +37 -0
- package/deps/gumbo-parser/src/char_ref.c +301 -0
- package/deps/gumbo-parser/src/char_ref.h +70 -0
- package/deps/gumbo-parser/src/char_ref_gperf.c +11126 -0
- package/deps/gumbo-parser/src/error.c +287 -0
- package/deps/gumbo-parser/src/error.h +225 -0
- package/deps/gumbo-parser/src/gumbo.h +683 -0
- package/deps/gumbo-parser/src/insertion_mode.h +55 -0
- package/deps/gumbo-parser/src/parser.c +4433 -0
- package/deps/gumbo-parser/src/parser.h +57 -0
- package/deps/gumbo-parser/src/string_buffer.c +110 -0
- package/deps/gumbo-parser/src/string_buffer.h +84 -0
- package/deps/gumbo-parser/src/string_piece.c +48 -0
- package/deps/gumbo-parser/src/string_piece.h +38 -0
- package/deps/gumbo-parser/src/tag.c +125 -0
- package/deps/gumbo-parser/src/tag_enum.h +155 -0
- package/deps/gumbo-parser/src/tag_gperf.h +350 -0
- package/deps/gumbo-parser/src/tag_sizes.h +2 -0
- package/deps/gumbo-parser/src/tag_strings.h +155 -0
- package/deps/gumbo-parser/src/token_type.h +42 -0
- package/deps/gumbo-parser/src/tokenizer.c +3003 -0
- package/deps/gumbo-parser/src/tokenizer.h +123 -0
- package/deps/gumbo-parser/src/tokenizer_states.h +104 -0
- package/deps/gumbo-parser/src/utf8.c +270 -0
- package/deps/gumbo-parser/src/utf8.h +132 -0
- package/deps/gumbo-parser/src/util.c +58 -0
- package/deps/gumbo-parser/src/util.h +60 -0
- package/deps/gumbo-parser/src/vector.c +128 -0
- package/deps/gumbo-parser/src/vector.h +70 -0
- package/dist/cjs/index.d.ts.map +1 -1
- package/dist/cjs/index.js +6 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +6 -0
- package/package.json +11 -3
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
#include <fstream>
|
|
2
|
+
#include <iostream>
|
|
3
|
+
#include <stdlib.h>
|
|
4
|
+
#include <string>
|
|
5
|
+
#include <algorithm>
|
|
6
|
+
#include <sstream>
|
|
7
|
+
|
|
8
|
+
#include "gumbo.h"
|
|
9
|
+
#include "html_parser.h"
|
|
10
|
+
|
|
11
|
+
// Helper: Converts hex characters to integer values
|
|
12
|
+
static int hexValue(char ch) {
|
|
13
|
+
if (ch >= '0' && ch <= '9') return ch - '0';
|
|
14
|
+
if (ch >= 'A' && ch <= 'F') return ch - 'A' + 10;
|
|
15
|
+
if (ch >= 'a' && ch <= 'f') return ch - 'a' + 10;
|
|
16
|
+
return -1;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
// 1. Decodes Quoted-Printable emails so formatting/IDs don't break across lines
|
|
20
|
+
static std::string decodeQuotedPrintable(const std::string& input) {
|
|
21
|
+
std::ostringstream output;
|
|
22
|
+
for (size_t i = 0; i < input.length(); ++i) {
|
|
23
|
+
if (input[i] == '=') {
|
|
24
|
+
if (i + 1 < input.length() && input[i + 1] == '\r') {
|
|
25
|
+
if (i + 2 < input.length() && input[i + 2] == '\n') { i += 2; continue; }
|
|
26
|
+
i += 1; continue;
|
|
27
|
+
}
|
|
28
|
+
if (i + 1 < input.length() && input[i + 1] == '\n') { i += 1; continue; }
|
|
29
|
+
|
|
30
|
+
if (i + 2 < input.length()) {
|
|
31
|
+
int high = hexValue(input[i + 1]);
|
|
32
|
+
int low = hexValue(input[i + 2]);
|
|
33
|
+
if (high != -1 && low != -1) {
|
|
34
|
+
output << (char)((high << 4) | low);
|
|
35
|
+
i += 2;
|
|
36
|
+
continue;
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
output << input[i];
|
|
41
|
+
}
|
|
42
|
+
return output.str();
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
static bool hasAttributeValue(const GumboElement* element, const std::string& attr_name, const std::string& target_value, bool partial_match = false) {
|
|
46
|
+
GumboAttribute* attr = gumbo_get_attribute(&element->attributes, attr_name.c_str());
|
|
47
|
+
if (!attr) return false;
|
|
48
|
+
|
|
49
|
+
std::string value = attr->value;
|
|
50
|
+
value.erase(std::remove(value.begin(), value.end(), '\r'), value.end());
|
|
51
|
+
value.erase(std::remove(value.begin(), value.end(), '\n'), value.end());
|
|
52
|
+
std::transform(value.begin(), value.end(), value.begin(), ::tolower);
|
|
53
|
+
|
|
54
|
+
if (partial_match) {
|
|
55
|
+
return value.find(target_value) != std::string::npos;
|
|
56
|
+
}
|
|
57
|
+
return value == target_value;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
static bool isSignatureNode(const GumboElement* element) {
|
|
61
|
+
if (element->tag == GUMBO_TAG_DIV) {
|
|
62
|
+
if (hasAttributeValue(element, "id", "signature", true) ||
|
|
63
|
+
hasAttributeValue(element, "class", "signature", true) ||
|
|
64
|
+
hasAttributeValue(element, "class", "gmail_signature", true)) {
|
|
65
|
+
return true;
|
|
66
|
+
}
|
|
67
|
+
} else if (element->tag == GUMBO_TAG_SPAN) {
|
|
68
|
+
if (hasAttributeValue(element, "class", "gmail_signature_prefix", true) ||
|
|
69
|
+
hasAttributeValue(element, "class", "gmail_signature", true)) {
|
|
70
|
+
return true;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
return false;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
static bool isReplyNodeAndFollowing(const GumboElement* element) {
|
|
77
|
+
if (element->tag == GUMBO_TAG_DIV) {
|
|
78
|
+
if (hasAttributeValue(element, "class", "gmail_quote", true) ||
|
|
79
|
+
hasAttributeValue(element, "id", "divrplyfwdmsg", false) ||
|
|
80
|
+
hasAttributeValue(element, "id", "quote", true) ||
|
|
81
|
+
hasAttributeValue(element, "id", "reply", true) ||
|
|
82
|
+
hasAttributeValue(element, "class", "quote", true) ||
|
|
83
|
+
hasAttributeValue(element, "class", "reply", true)) {
|
|
84
|
+
return true;
|
|
85
|
+
}
|
|
86
|
+
} else if (element->tag == GUMBO_TAG_BLOCKQUOTE) {
|
|
87
|
+
if (hasAttributeValue(element, "type", "cite", false)) {
|
|
88
|
+
return true;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
return false;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
static bool isGmailChip(const GumboElement* element) {
|
|
95
|
+
return hasAttributeValue(element, "class", "gmail_chip", true);
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
// 3. Rebuilds the DOM node framework into functional HTML
|
|
99
|
+
static std::string rebuildHtml(GumboNode* node, bool remove_signatures = false, bool remove_replies = false, bool remove_dividers = false, bool* stop_rendering = nullptr) {
|
|
100
|
+
if (stop_rendering && *stop_rendering) return "";
|
|
101
|
+
|
|
102
|
+
if (node->type == GUMBO_NODE_TEXT) {
|
|
103
|
+
std::string text = node->v.text.text;
|
|
104
|
+
if (remove_dividers && stop_rendering) {
|
|
105
|
+
// Manual scan for divider pattern: 5+ hyphens, any text, 5+ hyphens
|
|
106
|
+
const char* str = text.c_str();
|
|
107
|
+
size_t len = text.size();
|
|
108
|
+
for (size_t i = 0; i < len; ++i) {
|
|
109
|
+
if (str[i] != '-') continue;
|
|
110
|
+
// Count leading hyphens
|
|
111
|
+
size_t hyphenStart = i;
|
|
112
|
+
while (i < len && str[i] == '-') ++i;
|
|
113
|
+
size_t firstCount = i - hyphenStart;
|
|
114
|
+
if (firstCount < 5) continue;
|
|
115
|
+
// Scan for trailing 5+ hyphens
|
|
116
|
+
for (size_t j = i; j < len; ++j) {
|
|
117
|
+
if (str[j] != '-') continue;
|
|
118
|
+
size_t trailStart = j;
|
|
119
|
+
while (j < len && str[j] == '-') ++j;
|
|
120
|
+
if (j - trailStart >= 5) {
|
|
121
|
+
// Found divider: truncate at the start of first hyphen run
|
|
122
|
+
*stop_rendering = true;
|
|
123
|
+
return text.substr(0, hyphenStart);
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
// No trailing hyphens found after this run, continue scanning
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
return text;
|
|
130
|
+
}
|
|
131
|
+
if (node->type == GUMBO_NODE_ELEMENT) {
|
|
132
|
+
std::string html = "";
|
|
133
|
+
GumboElement* element = &node->v.element;
|
|
134
|
+
|
|
135
|
+
if (remove_signatures && isSignatureNode(element)) {
|
|
136
|
+
return "";
|
|
137
|
+
}
|
|
138
|
+
if (remove_replies && isGmailChip(element)) {
|
|
139
|
+
return "";
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
const char* tag_name = gumbo_normalized_tagname(element->tag);
|
|
143
|
+
bool write_tags = (element->tag != GUMBO_TAG_UNKNOWN);
|
|
144
|
+
|
|
145
|
+
if (write_tags) {
|
|
146
|
+
html += "<" + std::string(tag_name);
|
|
147
|
+
for (unsigned int i = 0; i < element->attributes.length; ++i) {
|
|
148
|
+
GumboAttribute* attr = (GumboAttribute*)element->attributes.data[i];
|
|
149
|
+
html += " " + std::string(attr->name) + "=\"" + std::string(attr->value) + "\"";
|
|
150
|
+
}
|
|
151
|
+
html += ">";
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
GumboVector* children = &element->children;
|
|
155
|
+
for (unsigned int i = 0; i < children->length; ++i) {
|
|
156
|
+
if (stop_rendering && *stop_rendering) break;
|
|
157
|
+
|
|
158
|
+
GumboNode* child = (GumboNode*)children->data[i];
|
|
159
|
+
if (remove_replies && child->type == GUMBO_NODE_ELEMENT) {
|
|
160
|
+
if (isReplyNodeAndFollowing(&child->v.element)) {
|
|
161
|
+
break; // Stop rendering this child and all following siblings
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
html += rebuildHtml(child, remove_signatures, remove_replies, remove_dividers, stop_rendering);
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
if (write_tags && element->tag != GUMBO_TAG_AREA && element->tag != GUMBO_TAG_BASE &&
|
|
168
|
+
element->tag != GUMBO_TAG_BR && element->tag != GUMBO_TAG_IMG && element->tag != GUMBO_TAG_INPUT) {
|
|
169
|
+
html += "</" + std::string(tag_name) + ">";
|
|
170
|
+
}
|
|
171
|
+
return html;
|
|
172
|
+
}
|
|
173
|
+
return "";
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
namespace liteEmailParser {
|
|
177
|
+
|
|
178
|
+
std::string removeSignature(std::string html) {
|
|
179
|
+
// Clean email transmission artifacts first
|
|
180
|
+
std::string decodedHtml = decodeQuotedPrintable(html);
|
|
181
|
+
|
|
182
|
+
GumboOutput* output = gumbo_parse(decodedHtml.c_str());
|
|
183
|
+
std::string cleanedHtml = rebuildHtml(output->root, true, false);
|
|
184
|
+
gumbo_destroy_output(&kGumboDefaultOptions, output);
|
|
185
|
+
|
|
186
|
+
return cleanedHtml;
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
std::string removeReplies(std::string html) {
|
|
190
|
+
std::string decodedHtml = decodeQuotedPrintable(html);
|
|
191
|
+
|
|
192
|
+
GumboOutput* output = gumbo_parse(decodedHtml.c_str());
|
|
193
|
+
std::string cleanedHtml = rebuildHtml(output->root, false, true);
|
|
194
|
+
gumbo_destroy_output(&kGumboDefaultOptions, output);
|
|
195
|
+
|
|
196
|
+
return cleanedHtml;
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
std::string removeDividers(std::string html) {
|
|
200
|
+
std::string decodedHtml = decodeQuotedPrintable(html);
|
|
201
|
+
|
|
202
|
+
GumboOutput* output = gumbo_parse(decodedHtml.c_str());
|
|
203
|
+
bool stop_rendering = false;
|
|
204
|
+
std::string cleanedHtml = rebuildHtml(output->root, false, false, true, &stop_rendering);
|
|
205
|
+
gumbo_destroy_output(&kGumboDefaultOptions, output);
|
|
206
|
+
|
|
207
|
+
return cleanedHtml;
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
std::string cleanHtml(std::string html) {
|
|
211
|
+
std::string decodedHtml = decodeQuotedPrintable(html);
|
|
212
|
+
|
|
213
|
+
GumboOutput* output = gumbo_parse(decodedHtml.c_str());
|
|
214
|
+
bool stop_rendering = false;
|
|
215
|
+
std::string cleanedHtml = rebuildHtml(output->root, true, true, true, &stop_rendering);
|
|
216
|
+
gumbo_destroy_output(&kGumboDefaultOptions, output);
|
|
217
|
+
|
|
218
|
+
return cleanedHtml;
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
} // namespace liteEmailParser
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
// Copyright 2010 Google Inc. All Rights Reserved.
|
|
2
|
+
//
|
|
3
|
+
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
// you may not use this file except in compliance with the License.
|
|
5
|
+
// You may obtain a copy of the License at
|
|
6
|
+
//
|
|
7
|
+
// http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
//
|
|
9
|
+
// Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
// See the License for the specific language governing permissions and
|
|
13
|
+
// limitations under the License.
|
|
14
|
+
//
|
|
15
|
+
// Author: jdtang@google.com (Jonathan Tang)
|
|
16
|
+
|
|
17
|
+
#include "attribute.h"
|
|
18
|
+
|
|
19
|
+
#include <assert.h>
|
|
20
|
+
#include <stdlib.h>
|
|
21
|
+
#include <string.h>
|
|
22
|
+
#include <strings.h>
|
|
23
|
+
|
|
24
|
+
#include "util.h"
|
|
25
|
+
|
|
26
|
+
struct GumboInternalParser;
|
|
27
|
+
|
|
28
|
+
GumboAttribute* gumbo_get_attribute(
|
|
29
|
+
const GumboVector* attributes, const char* name) {
|
|
30
|
+
for (unsigned int i = 0; i < attributes->length; ++i) {
|
|
31
|
+
GumboAttribute* attr = attributes->data[i];
|
|
32
|
+
if (!strcasecmp(attr->name, name)) {
|
|
33
|
+
return attr;
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
return NULL;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
void gumbo_destroy_attribute(
|
|
40
|
+
struct GumboInternalParser* parser, GumboAttribute* attribute) {
|
|
41
|
+
gumbo_parser_deallocate(parser, (void*) attribute->name);
|
|
42
|
+
gumbo_parser_deallocate(parser, (void*) attribute->value);
|
|
43
|
+
gumbo_parser_deallocate(parser, (void*) attribute);
|
|
44
|
+
}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
// Copyright 2010 Google Inc. All Rights Reserved.
|
|
2
|
+
//
|
|
3
|
+
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
// you may not use this file except in compliance with the License.
|
|
5
|
+
// You may obtain a copy of the License at
|
|
6
|
+
//
|
|
7
|
+
// http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
//
|
|
9
|
+
// Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
// See the License for the specific language governing permissions and
|
|
13
|
+
// limitations under the License.
|
|
14
|
+
//
|
|
15
|
+
// Author: jdtang@google.com (Jonathan Tang)
|
|
16
|
+
|
|
17
|
+
#ifndef GUMBO_ATTRIBUTE_H_
|
|
18
|
+
#define GUMBO_ATTRIBUTE_H_
|
|
19
|
+
|
|
20
|
+
#include "gumbo.h"
|
|
21
|
+
|
|
22
|
+
#ifdef __cplusplus
|
|
23
|
+
extern "C" {
|
|
24
|
+
#endif
|
|
25
|
+
|
|
26
|
+
struct GumboInternalParser;
|
|
27
|
+
|
|
28
|
+
// Release the memory used for an GumboAttribute, including the attribute
|
|
29
|
+
// itself.
|
|
30
|
+
void gumbo_destroy_attribute(
|
|
31
|
+
struct GumboInternalParser* parser, GumboAttribute* attribute);
|
|
32
|
+
|
|
33
|
+
#ifdef __cplusplus
|
|
34
|
+
}
|
|
35
|
+
#endif
|
|
36
|
+
|
|
37
|
+
#endif // GUMBO_ATTRIBUTE_H_
|
|
@@ -0,0 +1,301 @@
|
|
|
1
|
+
// Copyright 2011 Google Inc. All Rights Reserved.
|
|
2
|
+
//
|
|
3
|
+
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
// you may not use this file except in compliance with the License.
|
|
5
|
+
// You may obtain a copy of the License at
|
|
6
|
+
//
|
|
7
|
+
// http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
//
|
|
9
|
+
// Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
// See the License for the specific language governing permissions and
|
|
13
|
+
// limitations under the License.
|
|
14
|
+
//
|
|
15
|
+
// Author: jdtang@google.com (Jonathan Tang)
|
|
16
|
+
|
|
17
|
+
#include "char_ref.h"
|
|
18
|
+
|
|
19
|
+
#include <assert.h>
|
|
20
|
+
#include <ctype.h>
|
|
21
|
+
#include <stddef.h>
|
|
22
|
+
#include <stdio.h>
|
|
23
|
+
#include <string.h> // Only for debug assertions at present.
|
|
24
|
+
|
|
25
|
+
#include "error.h"
|
|
26
|
+
#include "string_piece.h"
|
|
27
|
+
#include "utf8.h"
|
|
28
|
+
#include "util.h"
|
|
29
|
+
|
|
30
|
+
enum {
|
|
31
|
+
GUMBO_NAMED_CHAR_REF_MAX_LEN = 32
|
|
32
|
+
};
|
|
33
|
+
|
|
34
|
+
struct GumboInternalParser;
|
|
35
|
+
|
|
36
|
+
const int kGumboNoChar = -1;
|
|
37
|
+
|
|
38
|
+
// Table of replacement characters. The spec specifies that any occurrence of
|
|
39
|
+
// the first character should be replaced by the second character, and a parse
|
|
40
|
+
// error recorded.
|
|
41
|
+
typedef struct {
|
|
42
|
+
int from_char;
|
|
43
|
+
int to_char;
|
|
44
|
+
} CharReplacement;
|
|
45
|
+
|
|
46
|
+
static const CharReplacement kCharReplacements[] = {{0x00, 0xfffd},
|
|
47
|
+
{0x0d, 0x000d}, {0x80, 0x20ac}, {0x81, 0x0081}, {0x82, 0x201A},
|
|
48
|
+
{0x83, 0x0192}, {0x84, 0x201E}, {0x85, 0x2026}, {0x86, 0x2020},
|
|
49
|
+
{0x87, 0x2021}, {0x88, 0x02C6}, {0x89, 0x2030}, {0x8A, 0x0160},
|
|
50
|
+
{0x8B, 0x2039}, {0x8C, 0x0152}, {0x8D, 0x008D}, {0x8E, 0x017D},
|
|
51
|
+
{0x8F, 0x008F}, {0x90, 0x0090}, {0x91, 0x2018}, {0x92, 0x2019},
|
|
52
|
+
{0x93, 0x201C}, {0x94, 0x201D}, {0x95, 0x2022}, {0x96, 0x2013},
|
|
53
|
+
{0x97, 0x2014}, {0x98, 0x02DC}, {0x99, 0x2122}, {0x9A, 0x0161},
|
|
54
|
+
{0x9B, 0x203A}, {0x9C, 0x0153}, {0x9D, 0x009D}, {0x9E, 0x017E},
|
|
55
|
+
{0x9F, 0x0178},
|
|
56
|
+
// Terminator.
|
|
57
|
+
{-1, -1}};
|
|
58
|
+
|
|
59
|
+
static int parse_digit(int c, bool allow_hex) {
|
|
60
|
+
if (c >= '0' && c <= '9') {
|
|
61
|
+
return c - '0';
|
|
62
|
+
}
|
|
63
|
+
if (allow_hex && c >= 'a' && c <= 'f') {
|
|
64
|
+
return c - 'a' + 10;
|
|
65
|
+
}
|
|
66
|
+
if (allow_hex && c >= 'A' && c <= 'F') {
|
|
67
|
+
return c - 'A' + 10;
|
|
68
|
+
}
|
|
69
|
+
return -1;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
static void add_no_digit_error(
|
|
73
|
+
struct GumboInternalParser* parser, Utf8Iterator* input) {
|
|
74
|
+
GumboError* error = gumbo_add_error(parser);
|
|
75
|
+
if (!error) {
|
|
76
|
+
return;
|
|
77
|
+
}
|
|
78
|
+
utf8iterator_fill_error_at_mark(input, error);
|
|
79
|
+
error->type = GUMBO_ERR_NUMERIC_CHAR_REF_NO_DIGITS;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
static void add_codepoint_error(struct GumboInternalParser* parser,
|
|
83
|
+
Utf8Iterator* input, GumboErrorType type, int codepoint) {
|
|
84
|
+
GumboError* error = gumbo_add_error(parser);
|
|
85
|
+
if (!error) {
|
|
86
|
+
return;
|
|
87
|
+
}
|
|
88
|
+
utf8iterator_fill_error_at_mark(input, error);
|
|
89
|
+
error->type = type;
|
|
90
|
+
error->v.codepoint = codepoint;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
static void add_named_reference_error(struct GumboInternalParser* parser,
|
|
94
|
+
Utf8Iterator* input, GumboErrorType type, GumboStringPiece text) {
|
|
95
|
+
GumboError* error = gumbo_add_error(parser);
|
|
96
|
+
if (!error) {
|
|
97
|
+
return;
|
|
98
|
+
}
|
|
99
|
+
utf8iterator_fill_error_at_mark(input, error);
|
|
100
|
+
error->type = type;
|
|
101
|
+
error->v.text = text;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
static int maybe_replace_codepoint(int codepoint) {
|
|
105
|
+
for (int i = 0; kCharReplacements[i].from_char != -1; ++i) {
|
|
106
|
+
if (kCharReplacements[i].from_char == codepoint) {
|
|
107
|
+
return kCharReplacements[i].to_char;
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
return -1;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
static bool consume_numeric_ref(
|
|
114
|
+
struct GumboInternalParser* parser, Utf8Iterator* input, int* output) {
|
|
115
|
+
utf8iterator_next(input);
|
|
116
|
+
bool is_hex = false;
|
|
117
|
+
int c = utf8iterator_current(input);
|
|
118
|
+
if (c == 'x' || c == 'X') {
|
|
119
|
+
is_hex = true;
|
|
120
|
+
utf8iterator_next(input);
|
|
121
|
+
c = utf8iterator_current(input);
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
int digit = parse_digit(c, is_hex);
|
|
125
|
+
if (digit == -1) {
|
|
126
|
+
// First digit was invalid; add a parse error and return.
|
|
127
|
+
add_no_digit_error(parser, input);
|
|
128
|
+
utf8iterator_reset(input);
|
|
129
|
+
*output = kGumboNoChar;
|
|
130
|
+
return false;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
unsigned int codepoint = 0;
|
|
134
|
+
bool status = true;
|
|
135
|
+
do {
|
|
136
|
+
codepoint = (codepoint * (is_hex ? 16 : 10)) + digit;
|
|
137
|
+
utf8iterator_next(input);
|
|
138
|
+
digit = parse_digit(utf8iterator_current(input), is_hex);
|
|
139
|
+
} while (digit != -1);
|
|
140
|
+
|
|
141
|
+
if (utf8iterator_current(input) != ';') {
|
|
142
|
+
add_codepoint_error(
|
|
143
|
+
parser, input, GUMBO_ERR_NUMERIC_CHAR_REF_WITHOUT_SEMICOLON, codepoint);
|
|
144
|
+
status = false;
|
|
145
|
+
} else {
|
|
146
|
+
utf8iterator_next(input);
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
int replacement = maybe_replace_codepoint(codepoint);
|
|
150
|
+
if (replacement != -1) {
|
|
151
|
+
add_codepoint_error(
|
|
152
|
+
parser, input, GUMBO_ERR_NUMERIC_CHAR_REF_INVALID, codepoint);
|
|
153
|
+
*output = replacement;
|
|
154
|
+
return false;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
if ((codepoint >= 0xd800 && codepoint <= 0xdfff) || codepoint > 0x10ffff) {
|
|
158
|
+
add_codepoint_error(
|
|
159
|
+
parser, input, GUMBO_ERR_NUMERIC_CHAR_REF_INVALID, codepoint);
|
|
160
|
+
*output = 0xfffd;
|
|
161
|
+
return false;
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
if (utf8_is_invalid_code_point(codepoint) || codepoint == 0xb) {
|
|
165
|
+
add_codepoint_error(
|
|
166
|
+
parser, input, GUMBO_ERR_NUMERIC_CHAR_REF_INVALID, codepoint);
|
|
167
|
+
status = false;
|
|
168
|
+
// But return it anyway, per spec.
|
|
169
|
+
}
|
|
170
|
+
*output = codepoint;
|
|
171
|
+
return status;
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
static bool maybe_add_invalid_named_reference(
|
|
175
|
+
struct GumboInternalParser* parser, Utf8Iterator* input) {
|
|
176
|
+
// The iterator will always be reset in this code path, so we don't need to
|
|
177
|
+
// worry about consuming characters.
|
|
178
|
+
const char* start = utf8iterator_get_char_pointer(input);
|
|
179
|
+
int c = utf8iterator_current(input);
|
|
180
|
+
while ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') ||
|
|
181
|
+
(c >= '0' && c <= '9')) {
|
|
182
|
+
utf8iterator_next(input);
|
|
183
|
+
c = utf8iterator_current(input);
|
|
184
|
+
}
|
|
185
|
+
if (c == ';') {
|
|
186
|
+
GumboStringPiece bad_ref;
|
|
187
|
+
bad_ref.data = start;
|
|
188
|
+
bad_ref.length = utf8iterator_get_char_pointer(input) - start;
|
|
189
|
+
add_named_reference_error(
|
|
190
|
+
parser, input, GUMBO_ERR_NAMED_CHAR_REF_INVALID, bad_ref);
|
|
191
|
+
return false;
|
|
192
|
+
}
|
|
193
|
+
return true;
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
static const char* find_named_ref(
|
|
197
|
+
const char* start, const char* end, OneOrTwoCodepoints* output) {
|
|
198
|
+
const char* p = start;
|
|
199
|
+
const char* best_end = NULL;
|
|
200
|
+
OneOrTwoCodepoints best = { kGumboNoChar, kGumboNoChar };
|
|
201
|
+
|
|
202
|
+
while (p < end && (size_t) (p - start) < GUMBO_NAMED_CHAR_REF_MAX_LEN) {
|
|
203
|
+
const unsigned char c = (unsigned char) *p;
|
|
204
|
+
if (!isalnum(c) && c != ';') {
|
|
205
|
+
break;
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
++p;
|
|
209
|
+
|
|
210
|
+
const struct GumboNamedCharRef* ref = gumbo_named_char_ref_find(start, (size_t)(p - start));
|
|
211
|
+
if (ref) {
|
|
212
|
+
best.first = ref->first;
|
|
213
|
+
best.second = ref->second;
|
|
214
|
+
best_end = p;
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
if (c == ';') {
|
|
218
|
+
break;
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
if (!best_end) {
|
|
223
|
+
return NULL;
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
*output = best;
|
|
227
|
+
return best_end;
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
static bool consume_named_ref(
|
|
232
|
+
struct GumboInternalParser* parser, Utf8Iterator* input, bool is_in_attribute,
|
|
233
|
+
OneOrTwoCodepoints* output) {
|
|
234
|
+
assert(output->first == kGumboNoChar);
|
|
235
|
+
const char* start = utf8iterator_get_char_pointer(input);
|
|
236
|
+
const char* end = utf8iterator_get_end_pointer(input);
|
|
237
|
+
const char* match_end = find_named_ref(start, end, output);
|
|
238
|
+
|
|
239
|
+
if (!match_end) {
|
|
240
|
+
output->first = kGumboNoChar;
|
|
241
|
+
output->second = kGumboNoChar;
|
|
242
|
+
bool status = maybe_add_invalid_named_reference(parser, input);
|
|
243
|
+
utf8iterator_reset(input);
|
|
244
|
+
return status;
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
assert(output->first != kGumboNoChar);
|
|
248
|
+
char last_char = *(match_end - 1);
|
|
249
|
+
int len = match_end - start;
|
|
250
|
+
if (last_char == ';') {
|
|
251
|
+
bool matched = utf8iterator_maybe_consume_match(input, start, len, true);
|
|
252
|
+
assert(matched);
|
|
253
|
+
return true;
|
|
254
|
+
} else if (is_in_attribute &&
|
|
255
|
+
match_end < end &&
|
|
256
|
+
(*match_end == '=' || isalnum((unsigned char) *match_end))) {
|
|
257
|
+
output->first = kGumboNoChar;
|
|
258
|
+
output->second = kGumboNoChar;
|
|
259
|
+
utf8iterator_reset(input);
|
|
260
|
+
return true;
|
|
261
|
+
} else {
|
|
262
|
+
GumboStringPiece bad_ref;
|
|
263
|
+
bad_ref.length = match_end - start;
|
|
264
|
+
bad_ref.data = start;
|
|
265
|
+
add_named_reference_error(
|
|
266
|
+
parser, input, GUMBO_ERR_NAMED_CHAR_REF_WITHOUT_SEMICOLON, bad_ref);
|
|
267
|
+
bool matched = utf8iterator_maybe_consume_match(input, start, len, true);
|
|
268
|
+
assert(matched);
|
|
269
|
+
return false;
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
bool consume_char_ref(struct GumboInternalParser* parser,
|
|
274
|
+
struct GumboInternalUtf8Iterator* input, int additional_allowed_char,
|
|
275
|
+
bool is_in_attribute, OneOrTwoCodepoints* output) {
|
|
276
|
+
utf8iterator_mark(input);
|
|
277
|
+
utf8iterator_next(input);
|
|
278
|
+
int c = utf8iterator_current(input);
|
|
279
|
+
output->first = kGumboNoChar;
|
|
280
|
+
output->second = kGumboNoChar;
|
|
281
|
+
if (c == additional_allowed_char) {
|
|
282
|
+
utf8iterator_reset(input);
|
|
283
|
+
output->first = kGumboNoChar;
|
|
284
|
+
return true;
|
|
285
|
+
}
|
|
286
|
+
switch (utf8iterator_current(input)) {
|
|
287
|
+
case '\t':
|
|
288
|
+
case '\n':
|
|
289
|
+
case '\f':
|
|
290
|
+
case ' ':
|
|
291
|
+
case '<':
|
|
292
|
+
case '&':
|
|
293
|
+
case -1:
|
|
294
|
+
utf8iterator_reset(input);
|
|
295
|
+
return true;
|
|
296
|
+
case '#':
|
|
297
|
+
return consume_numeric_ref(parser, input, &output->first);
|
|
298
|
+
default:
|
|
299
|
+
return consume_named_ref(parser, input, is_in_attribute, output);
|
|
300
|
+
}
|
|
301
|
+
}
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
// Copyright 2011 Google Inc. All Rights Reserved.
|
|
2
|
+
//
|
|
3
|
+
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
// you may not use this file except in compliance with the License.
|
|
5
|
+
// You may obtain a copy of the License at
|
|
6
|
+
//
|
|
7
|
+
// http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
//
|
|
9
|
+
// Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
// See the License for the specific language governing permissions and
|
|
13
|
+
// limitations under the License.
|
|
14
|
+
//
|
|
15
|
+
// Author: jdtang@google.com (Jonathan Tang)
|
|
16
|
+
//
|
|
17
|
+
// Internal header for character reference handling; this should not be exposed
|
|
18
|
+
// transitively by any public API header. This is why the functions aren't
|
|
19
|
+
// namespaced.
|
|
20
|
+
|
|
21
|
+
#ifndef GUMBO_CHAR_REF_H_
|
|
22
|
+
#define GUMBO_CHAR_REF_H_
|
|
23
|
+
|
|
24
|
+
#include <stdbool.h>
|
|
25
|
+
#include <stddef.h>
|
|
26
|
+
|
|
27
|
+
#ifdef __cplusplus
|
|
28
|
+
extern "C" {
|
|
29
|
+
#endif
|
|
30
|
+
|
|
31
|
+
struct GumboInternalParser;
|
|
32
|
+
struct GumboInternalUtf8Iterator;
|
|
33
|
+
|
|
34
|
+
// Value that indicates no character was produced.
|
|
35
|
+
extern const int kGumboNoChar;
|
|
36
|
+
|
|
37
|
+
// Certain named character references generate two codepoints, not one, and so
|
|
38
|
+
// the consume_char_ref subroutine needs to return this instead of an int. The
|
|
39
|
+
// first field will be kGumboNoChar if no character reference was found; the
|
|
40
|
+
// second field will be kGumboNoChar if that is the case or if the character
|
|
41
|
+
// reference returns only a single codepoint.
|
|
42
|
+
typedef struct {
|
|
43
|
+
int first;
|
|
44
|
+
int second;
|
|
45
|
+
} OneOrTwoCodepoints;
|
|
46
|
+
|
|
47
|
+
// Implements the "consume a character reference" section of the spec.
|
|
48
|
+
// This reads in characters from the input as necessary, and fills in a
|
|
49
|
+
// OneOrTwoCodepoints struct containing the characters read. It may add parse
|
|
50
|
+
// errors to the GumboParser's errors vector, if the spec calls for it. Pass a
|
|
51
|
+
// space for the "additional allowed char" when the spec says "with no
|
|
52
|
+
// additional allowed char". Returns false on parse error, true otherwise.
|
|
53
|
+
bool consume_char_ref(struct GumboInternalParser* parser,
|
|
54
|
+
struct GumboInternalUtf8Iterator* input, int additional_allowed_char,
|
|
55
|
+
bool is_in_attribute, OneOrTwoCodepoints* output);
|
|
56
|
+
|
|
57
|
+
struct GumboNamedCharRef {
|
|
58
|
+
int name;
|
|
59
|
+
int first;
|
|
60
|
+
int second;
|
|
61
|
+
};
|
|
62
|
+
|
|
63
|
+
const struct GumboNamedCharRef* gumbo_named_char_ref_find(
|
|
64
|
+
const char* str, size_t len);
|
|
65
|
+
|
|
66
|
+
#ifdef __cplusplus
|
|
67
|
+
}
|
|
68
|
+
#endif
|
|
69
|
+
|
|
70
|
+
#endif // GUMBO_CHAR_REF_H_
|