html-to-markdown 3.4.0 → 3.6.0.pre.rc.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/LICENSE +21 -0
- data/README.md +347 -0
- data/Steepfile +10 -2
- data/ext/html_to_markdown_rb/Cargo.toml +3 -2
- data/ext/html_to_markdown_rb/extconf.rb +5 -5
- data/ext/html_to_markdown_rb/native/Cargo.lock +962 -0
- data/ext/html_to_markdown_rb/native/Cargo.toml +6 -11
- data/ext/html_to_markdown_rb/native/extconf.rb +14 -0
- data/ext/html_to_markdown_rb/src/lib.rs +1715 -646
- data/lib/html_to_markdown/native.rb +913 -37
- data/lib/html_to_markdown/version.rb +3 -3
- data/lib/html_to_markdown.rb +9 -4
- data/lib/html_to_markdown_rb.so +0 -0
- data/sig/types.rbs +59 -292
- metadata +32 -179
- data/ext/html_to_markdown_rb/Makefile +0 -592
- data/lib/bin/html-to-markdown +0 -0
- data/vendor/Cargo.toml +0 -33
- data/vendor/html-to-markdown-rs/Cargo.toml +0 -54
- data/vendor/html-to-markdown-rs/README.md +0 -278
- data/vendor/html-to-markdown-rs/examples/basic.rs +0 -24
- data/vendor/html-to-markdown-rs/examples/table.rs +0 -25
- data/vendor/html-to-markdown-rs/examples/test_deser.rs +0 -12
- data/vendor/html-to-markdown-rs/examples/test_escape.rs +0 -58
- data/vendor/html-to-markdown-rs/examples/test_inline_formatting.rs +0 -113
- data/vendor/html-to-markdown-rs/examples/test_lists.rs +0 -39
- data/vendor/html-to-markdown-rs/examples/test_semantic_tags.rs +0 -89
- data/vendor/html-to-markdown-rs/examples/test_tables.rs +0 -100
- data/vendor/html-to-markdown-rs/examples/test_task_lists.rs +0 -61
- data/vendor/html-to-markdown-rs/examples/test_whitespace.rs +0 -34
- data/vendor/html-to-markdown-rs/src/convert_api.rs +0 -349
- data/vendor/html-to-markdown-rs/src/converter/block/blockquote.rs +0 -178
- data/vendor/html-to-markdown-rs/src/converter/block/container.rs +0 -114
- data/vendor/html-to-markdown-rs/src/converter/block/div.rs +0 -149
- data/vendor/html-to-markdown-rs/src/converter/block/heading.rs +0 -428
- data/vendor/html-to-markdown-rs/src/converter/block/horizontal_rule.rs +0 -103
- data/vendor/html-to-markdown-rs/src/converter/block/line_break.rs +0 -89
- data/vendor/html-to-markdown-rs/src/converter/block/mod.rs +0 -10
- data/vendor/html-to-markdown-rs/src/converter/block/paragraph.rs +0 -140
- data/vendor/html-to-markdown-rs/src/converter/block/preformatted.rs +0 -298
- data/vendor/html-to-markdown-rs/src/converter/block/table/builder.rs +0 -453
- data/vendor/html-to-markdown-rs/src/converter/block/table/caption.rs +0 -44
- data/vendor/html-to-markdown-rs/src/converter/block/table/cell.rs +0 -276
- data/vendor/html-to-markdown-rs/src/converter/block/table/cells.rs +0 -336
- data/vendor/html-to-markdown-rs/src/converter/block/table/layout.rs +0 -58
- data/vendor/html-to-markdown-rs/src/converter/block/table/mod.rs +0 -266
- data/vendor/html-to-markdown-rs/src/converter/block/table/scanner.rs +0 -146
- data/vendor/html-to-markdown-rs/src/converter/block/table/utils.rs +0 -34
- data/vendor/html-to-markdown-rs/src/converter/block/unknown.rs +0 -138
- data/vendor/html-to-markdown-rs/src/converter/context.rs +0 -208
- data/vendor/html-to-markdown-rs/src/converter/dom_context.rs +0 -337
- data/vendor/html-to-markdown-rs/src/converter/form/elements.rs +0 -770
- data/vendor/html-to-markdown-rs/src/converter/form/mod.rs +0 -82
- data/vendor/html-to-markdown-rs/src/converter/format/djot.rs +0 -64
- data/vendor/html-to-markdown-rs/src/converter/format/markdown.rs +0 -59
- data/vendor/html-to-markdown-rs/src/converter/format/mod.rs +0 -43
- data/vendor/html-to-markdown-rs/src/converter/handlers/blockquote.rs +0 -173
- data/vendor/html-to-markdown-rs/src/converter/handlers/code_block.rs +0 -434
- data/vendor/html-to-markdown-rs/src/converter/handlers/graphic.rs +0 -234
- data/vendor/html-to-markdown-rs/src/converter/handlers/image.rs +0 -282
- data/vendor/html-to-markdown-rs/src/converter/handlers/link.rs +0 -316
- data/vendor/html-to-markdown-rs/src/converter/handlers/mod.rs +0 -26
- data/vendor/html-to-markdown-rs/src/converter/inline/code.rs +0 -306
- data/vendor/html-to-markdown-rs/src/converter/inline/emphasis.rs +0 -345
- data/vendor/html-to-markdown-rs/src/converter/inline/link.rs +0 -428
- data/vendor/html-to-markdown-rs/src/converter/inline/mod.rs +0 -237
- data/vendor/html-to-markdown-rs/src/converter/inline/ruby.rs +0 -337
- data/vendor/html-to-markdown-rs/src/converter/inline/semantic/marks.rs +0 -566
- data/vendor/html-to-markdown-rs/src/converter/inline/semantic/mod.rs +0 -86
- data/vendor/html-to-markdown-rs/src/converter/inline/semantic/typography.rs +0 -558
- data/vendor/html-to-markdown-rs/src/converter/list/definition.rs +0 -232
- data/vendor/html-to-markdown-rs/src/converter/list/item.rs +0 -332
- data/vendor/html-to-markdown-rs/src/converter/list/mod.rs +0 -70
- data/vendor/html-to-markdown-rs/src/converter/list/ordered.rs +0 -201
- data/vendor/html-to-markdown-rs/src/converter/list/unordered.rs +0 -195
- data/vendor/html-to-markdown-rs/src/converter/list/utils.rs +0 -314
- data/vendor/html-to-markdown-rs/src/converter/main.rs +0 -710
- data/vendor/html-to-markdown-rs/src/converter/main_helpers.rs +0 -452
- data/vendor/html-to-markdown-rs/src/converter/media/embedded.rs +0 -393
- data/vendor/html-to-markdown-rs/src/converter/media/graphic.rs +0 -4
- data/vendor/html-to-markdown-rs/src/converter/media/image.rs +0 -183
- data/vendor/html-to-markdown-rs/src/converter/media/mod.rs +0 -87
- data/vendor/html-to-markdown-rs/src/converter/media/svg.rs +0 -280
- data/vendor/html-to-markdown-rs/src/converter/metadata.rs +0 -220
- data/vendor/html-to-markdown-rs/src/converter/mod.rs +0 -156
- data/vendor/html-to-markdown-rs/src/converter/plain_text.rs +0 -516
- data/vendor/html-to-markdown-rs/src/converter/preprocessing_helpers.rs +0 -201
- data/vendor/html-to-markdown-rs/src/converter/reference_collector.rs +0 -69
- data/vendor/html-to-markdown-rs/src/converter/semantic/attributes.rs +0 -269
- data/vendor/html-to-markdown-rs/src/converter/semantic/definition_list.rs +0 -266
- data/vendor/html-to-markdown-rs/src/converter/semantic/figure.rs +0 -391
- data/vendor/html-to-markdown-rs/src/converter/semantic/mod.rs +0 -112
- data/vendor/html-to-markdown-rs/src/converter/semantic/sectioning.rs +0 -85
- data/vendor/html-to-markdown-rs/src/converter/semantic/summary.rs +0 -324
- data/vendor/html-to-markdown-rs/src/converter/text/mod.rs +0 -8
- data/vendor/html-to-markdown-rs/src/converter/text/processing.rs +0 -56
- data/vendor/html-to-markdown-rs/src/converter/text_node.rs +0 -269
- data/vendor/html-to-markdown-rs/src/converter/utility/attributes.rs +0 -151
- data/vendor/html-to-markdown-rs/src/converter/utility/caching.rs +0 -74
- data/vendor/html-to-markdown-rs/src/converter/utility/content.rs +0 -271
- data/vendor/html-to-markdown-rs/src/converter/utility/mod.rs +0 -17
- data/vendor/html-to-markdown-rs/src/converter/utility/preprocessing.rs +0 -1002
- data/vendor/html-to-markdown-rs/src/converter/utility/serialization.rs +0 -126
- data/vendor/html-to-markdown-rs/src/converter/utility/siblings.rs +0 -97
- data/vendor/html-to-markdown-rs/src/converter/visitor_hooks.rs +0 -189
- data/vendor/html-to-markdown-rs/src/error.rs +0 -43
- data/vendor/html-to-markdown-rs/src/exports.rs +0 -24
- data/vendor/html-to-markdown-rs/src/inline_images.rs +0 -336
- data/vendor/html-to-markdown-rs/src/lib.rs +0 -139
- data/vendor/html-to-markdown-rs/src/metadata/collector.rs +0 -457
- data/vendor/html-to-markdown-rs/src/metadata/config.rs +0 -394
- data/vendor/html-to-markdown-rs/src/metadata/extraction.rs +0 -398
- data/vendor/html-to-markdown-rs/src/metadata/mod.rs +0 -288
- data/vendor/html-to-markdown-rs/src/metadata/types.rs +0 -477
- data/vendor/html-to-markdown-rs/src/options/conversion.rs +0 -559
- data/vendor/html-to-markdown-rs/src/options/inline_image.rs +0 -111
- data/vendor/html-to-markdown-rs/src/options/mod.rs +0 -20
- data/vendor/html-to-markdown-rs/src/options/preprocessing.rs +0 -201
- data/vendor/html-to-markdown-rs/src/options/validation.rs +0 -416
- data/vendor/html-to-markdown-rs/src/prelude.rs +0 -1
- data/vendor/html-to-markdown-rs/src/rcdom.rs +0 -487
- data/vendor/html-to-markdown-rs/src/text.rs +0 -358
- data/vendor/html-to-markdown-rs/src/types/document.rs +0 -191
- data/vendor/html-to-markdown-rs/src/types/mod.rs +0 -17
- data/vendor/html-to-markdown-rs/src/types/result.rs +0 -54
- data/vendor/html-to-markdown-rs/src/types/structure_builder.rs +0 -791
- data/vendor/html-to-markdown-rs/src/types/structure_collector.rs +0 -483
- data/vendor/html-to-markdown-rs/src/types/tables.rs +0 -52
- data/vendor/html-to-markdown-rs/src/types/warnings.rs +0 -33
- data/vendor/html-to-markdown-rs/src/validation.rs +0 -158
- data/vendor/html-to-markdown-rs/src/visitor/default_impl.rs +0 -63
- data/vendor/html-to-markdown-rs/src/visitor/mod.rs +0 -41
- data/vendor/html-to-markdown-rs/src/visitor/traits.rs +0 -370
- data/vendor/html-to-markdown-rs/src/visitor/types.rs +0 -319
- data/vendor/html-to-markdown-rs/src/visitor_helpers/helpers/callbacks/mod.rs +0 -1
- data/vendor/html-to-markdown-rs/src/visitor_helpers/helpers/content.rs +0 -126
- data/vendor/html-to-markdown-rs/src/visitor_helpers/helpers/mod.rs +0 -27
- data/vendor/html-to-markdown-rs/src/visitor_helpers/helpers/state.rs +0 -110
- data/vendor/html-to-markdown-rs/src/visitor_helpers/helpers/traversal.rs +0 -250
- data/vendor/html-to-markdown-rs/src/visitor_helpers.rs +0 -597
- data/vendor/html-to-markdown-rs/src/wrapper/sync.rs +0 -413
- data/vendor/html-to-markdown-rs/src/wrapper/utils.rs +0 -290
- data/vendor/html-to-markdown-rs/src/wrapper.rs +0 -9
- data/vendor/html-to-markdown-rs/tests/br_in_inline_test.rs +0 -87
- data/vendor/html-to-markdown-rs/tests/commonmark_compliance_test.rs +0 -297
- data/vendor/html-to-markdown-rs/tests/djot_output_test.rs +0 -153
- data/vendor/html-to-markdown-rs/tests/exclude_selectors_test.rs +0 -132
- data/vendor/html-to-markdown-rs/tests/integration_test.rs +0 -631
- data/vendor/html-to-markdown-rs/tests/issue_121_regressions.rs +0 -49
- data/vendor/html-to-markdown-rs/tests/issue_127_regressions.rs +0 -58
- data/vendor/html-to-markdown-rs/tests/issue_128_regressions.rs +0 -17
- data/vendor/html-to-markdown-rs/tests/issue_131_regressions.rs +0 -41
- data/vendor/html-to-markdown-rs/tests/issue_134_regressions.rs +0 -40
- data/vendor/html-to-markdown-rs/tests/issue_139_regressions.rs +0 -26
- data/vendor/html-to-markdown-rs/tests/issue_140_regressions.rs +0 -185
- data/vendor/html-to-markdown-rs/tests/issue_143_regressions.rs +0 -100
- data/vendor/html-to-markdown-rs/tests/issue_145_regressions.rs +0 -133
- data/vendor/html-to-markdown-rs/tests/issue_146_regressions.rs +0 -144
- data/vendor/html-to-markdown-rs/tests/issue_176_regressions.rs +0 -62
- data/vendor/html-to-markdown-rs/tests/issue_190_regressions.rs +0 -128
- data/vendor/html-to-markdown-rs/tests/issue_199_regressions.rs +0 -20
- data/vendor/html-to-markdown-rs/tests/issue_200_regressions.rs +0 -62
- data/vendor/html-to-markdown-rs/tests/issue_212_regressions.rs +0 -68
- data/vendor/html-to-markdown-rs/tests/issue_216_217_regressions.rs +0 -87
- data/vendor/html-to-markdown-rs/tests/issue_336_regressions.rs +0 -74
- data/vendor/html-to-markdown-rs/tests/issue_339_regressions.rs +0 -92
- data/vendor/html-to-markdown-rs/tests/issue_347_regressions.rs +0 -154
- data/vendor/html-to-markdown-rs/tests/issue_348_visitor_plain.rs +0 -93
- data/vendor/html-to-markdown-rs/tests/json_ld_script_extraction.rs +0 -44
- data/vendor/html-to-markdown-rs/tests/lists_test.rs +0 -199
- data/vendor/html-to-markdown-rs/tests/plain_output_test.rs +0 -273
- data/vendor/html-to-markdown-rs/tests/preprocessing_tests.rs +0 -61
- data/vendor/html-to-markdown-rs/tests/reference_links_test.rs +0 -169
- data/vendor/html-to-markdown-rs/tests/sectioning_elements_test.rs +0 -137
- data/vendor/html-to-markdown-rs/tests/skip_images_test.rs +0 -522
- data/vendor/html-to-markdown-rs/tests/tables_test.rs +0 -743
- data/vendor/html-to-markdown-rs/tests/test_custom_elements.rs +0 -41
- data/vendor/html-to-markdown-rs/tests/test_issue_187.rs +0 -204
- data/vendor/html-to-markdown-rs/tests/test_issue_218.rs +0 -68
- data/vendor/html-to-markdown-rs/tests/test_issue_277.rs +0 -77
- data/vendor/html-to-markdown-rs/tests/test_max_depth.rs +0 -82
- data/vendor/html-to-markdown-rs/tests/test_nested_simple.rs +0 -45
- data/vendor/html-to-markdown-rs/tests/test_script_style_stripping.rs +0 -396
- data/vendor/html-to-markdown-rs/tests/test_spa_bisect.rs +0 -34
- data/vendor/html-to-markdown-rs/tests/visitor_code_integration_test.rs +0 -121
- data/vendor/html-to-markdown-rs/tests/visitor_integration_test.rs +0 -1190
- data/vendor/html-to-markdown-rs/tests/xml_tables_test.rs +0 -372
|
@@ -1,398 +0,0 @@
|
|
|
1
|
-
//! Metadata extraction utilities and helpers.
|
|
2
|
-
|
|
3
|
-
use super::types::{DocumentMetadata, StructuredData, StructuredDataType, TextDirection};
|
|
4
|
-
use std::collections::BTreeMap;
|
|
5
|
-
|
|
6
|
-
/// Extract document metadata from collected head metadata.
|
|
7
|
-
///
|
|
8
|
-
/// Parses head metadata into structured document metadata,
|
|
9
|
-
/// handling special cases like Open Graph, Twitter Card, keywords, etc.
|
|
10
|
-
pub(crate) fn extract_document_metadata(
|
|
11
|
-
head_metadata: BTreeMap<String, String>,
|
|
12
|
-
lang: Option<String>,
|
|
13
|
-
dir: Option<String>,
|
|
14
|
-
) -> DocumentMetadata {
|
|
15
|
-
let mut doc = DocumentMetadata::default();
|
|
16
|
-
|
|
17
|
-
for (raw_key, value) in head_metadata {
|
|
18
|
-
let mut key = raw_key.as_str();
|
|
19
|
-
let mut replaced_key: Option<String> = None;
|
|
20
|
-
|
|
21
|
-
if let Some(stripped) = key.strip_prefix("meta-") {
|
|
22
|
-
key = stripped;
|
|
23
|
-
}
|
|
24
|
-
|
|
25
|
-
if key.as_bytes().contains(&b':') {
|
|
26
|
-
replaced_key = Some(key.replace(':', "-"));
|
|
27
|
-
key = replaced_key.as_deref().unwrap_or(key);
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
// Normalize to lowercase for case-insensitive matching (per HTML spec,
|
|
31
|
-
// meta name attributes are compared ASCII case-insensitively).
|
|
32
|
-
let lower_key = key.to_ascii_lowercase();
|
|
33
|
-
|
|
34
|
-
match lower_key.as_str() {
|
|
35
|
-
"title" => doc.title = Some(value),
|
|
36
|
-
"description" => doc.description = Some(value),
|
|
37
|
-
"author" | "creator" | "publisher" => {
|
|
38
|
-
if doc.author.is_none() {
|
|
39
|
-
doc.author = Some(value);
|
|
40
|
-
}
|
|
41
|
-
}
|
|
42
|
-
"canonical" => doc.canonical_url = Some(value),
|
|
43
|
-
"base" | "base-href" => doc.base_href = Some(value),
|
|
44
|
-
k if k.starts_with("og-") => {
|
|
45
|
-
let og_key = k.trim_start_matches("og-").replace('-', "_");
|
|
46
|
-
doc.open_graph.insert(og_key, value);
|
|
47
|
-
}
|
|
48
|
-
k if k.starts_with("twitter-") => {
|
|
49
|
-
let tw_key = k.trim_start_matches("twitter-").replace('-', "_");
|
|
50
|
-
doc.twitter_card.insert(tw_key, value);
|
|
51
|
-
}
|
|
52
|
-
// Dublin Core: DC.* and DCTERMS.* prefixes (dot becomes part of key after meta- strip).
|
|
53
|
-
// Map DC/DCTERMS fields to dedicated struct fields where applicable.
|
|
54
|
-
k if k.starts_with("dc.") || k.starts_with("dc-") => {
|
|
55
|
-
let dc_field = k.trim_start_matches("dc.").trim_start_matches("dc-");
|
|
56
|
-
match dc_field {
|
|
57
|
-
"title" => {
|
|
58
|
-
if doc.title.is_none() {
|
|
59
|
-
doc.title = Some(value);
|
|
60
|
-
}
|
|
61
|
-
}
|
|
62
|
-
"description" => {
|
|
63
|
-
if doc.description.is_none() {
|
|
64
|
-
doc.description = Some(value);
|
|
65
|
-
}
|
|
66
|
-
}
|
|
67
|
-
"creator" | "contributor" | "publisher" => {
|
|
68
|
-
if doc.author.is_none() {
|
|
69
|
-
doc.author = Some(value);
|
|
70
|
-
}
|
|
71
|
-
}
|
|
72
|
-
"subject" | "keywords" => {
|
|
73
|
-
if doc.keywords.is_empty() {
|
|
74
|
-
doc.keywords = split_keywords(&value);
|
|
75
|
-
}
|
|
76
|
-
}
|
|
77
|
-
_ => {
|
|
78
|
-
let meta_key = format!("dc_{}", dc_field.replace('-', "_"));
|
|
79
|
-
doc.meta_tags.insert(meta_key, value);
|
|
80
|
-
}
|
|
81
|
-
}
|
|
82
|
-
}
|
|
83
|
-
k if k.starts_with("dcterms.") || k.starts_with("dcterms-") => {
|
|
84
|
-
let dc_field = k.trim_start_matches("dcterms.").trim_start_matches("dcterms-");
|
|
85
|
-
match dc_field {
|
|
86
|
-
"title" | "alternative" => {
|
|
87
|
-
if doc.title.is_none() {
|
|
88
|
-
doc.title = Some(value);
|
|
89
|
-
}
|
|
90
|
-
}
|
|
91
|
-
"description" | "abstract" => {
|
|
92
|
-
if doc.description.is_none() {
|
|
93
|
-
doc.description = Some(value);
|
|
94
|
-
}
|
|
95
|
-
}
|
|
96
|
-
"creator" | "contributor" | "publisher" => {
|
|
97
|
-
if doc.author.is_none() {
|
|
98
|
-
doc.author = Some(value);
|
|
99
|
-
}
|
|
100
|
-
}
|
|
101
|
-
"subject" | "keywords" => {
|
|
102
|
-
if doc.keywords.is_empty() {
|
|
103
|
-
doc.keywords = split_keywords(&value);
|
|
104
|
-
}
|
|
105
|
-
}
|
|
106
|
-
_ => {
|
|
107
|
-
let meta_key = format!("dcterms_{}", dc_field.replace('-', "_"));
|
|
108
|
-
doc.meta_tags.insert(meta_key, value);
|
|
109
|
-
}
|
|
110
|
-
}
|
|
111
|
-
}
|
|
112
|
-
// All keyword-bearing meta tag variants
|
|
113
|
-
"keywords" | "news_keywords" | "citation_keywords" | "subject" | "topic" | "category"
|
|
114
|
-
| "classification" => {
|
|
115
|
-
if doc.keywords.is_empty() {
|
|
116
|
-
doc.keywords = split_keywords(&value);
|
|
117
|
-
}
|
|
118
|
-
}
|
|
119
|
-
_ => {
|
|
120
|
-
let meta_key = if key.as_ptr() == raw_key.as_ptr() && key.len() == raw_key.len() {
|
|
121
|
-
raw_key
|
|
122
|
-
} else if let Some(replaced) = replaced_key {
|
|
123
|
-
replaced
|
|
124
|
-
} else {
|
|
125
|
-
key.to_string()
|
|
126
|
-
};
|
|
127
|
-
doc.meta_tags.insert(meta_key, value);
|
|
128
|
-
}
|
|
129
|
-
}
|
|
130
|
-
}
|
|
131
|
-
|
|
132
|
-
if let Some(lang) = lang {
|
|
133
|
-
doc.language = Some(lang);
|
|
134
|
-
}
|
|
135
|
-
|
|
136
|
-
if let Some(dir) = dir {
|
|
137
|
-
if let Some(parsed_dir) = TextDirection::parse(&dir) {
|
|
138
|
-
doc.text_direction = Some(parsed_dir);
|
|
139
|
-
}
|
|
140
|
-
}
|
|
141
|
-
|
|
142
|
-
doc
|
|
143
|
-
}
|
|
144
|
-
|
|
145
|
-
/// Split a comma-separated keywords string into a `Vec<String>`.
|
|
146
|
-
fn split_keywords(value: &str) -> Vec<String> {
|
|
147
|
-
value
|
|
148
|
-
.split(',')
|
|
149
|
-
.map(|s| s.trim().to_string())
|
|
150
|
-
.filter(|s| !s.is_empty())
|
|
151
|
-
.collect()
|
|
152
|
-
}
|
|
153
|
-
|
|
154
|
-
/// Extract structured data blocks into `StructuredData` items.
|
|
155
|
-
pub(crate) fn extract_structured_data(json_ld: Vec<String>) -> Vec<StructuredData> {
|
|
156
|
-
let mut result = Vec::with_capacity(json_ld.len());
|
|
157
|
-
|
|
158
|
-
for json_str in json_ld {
|
|
159
|
-
let schema_type = scan_schema_type(&json_str)
|
|
160
|
-
.or_else(|| {
|
|
161
|
-
if json_str.contains("\"@type\"") {
|
|
162
|
-
serde_json::from_str::<serde_json::Value>(&json_str).ok().and_then(|v| {
|
|
163
|
-
v.get("@type")
|
|
164
|
-
.and_then(|t| t.as_str().map(std::string::ToString::to_string))
|
|
165
|
-
})
|
|
166
|
-
} else {
|
|
167
|
-
None
|
|
168
|
-
}
|
|
169
|
-
})
|
|
170
|
-
.or_else(|| {
|
|
171
|
-
if !json_str.contains("\"@graph\"") {
|
|
172
|
-
return None;
|
|
173
|
-
}
|
|
174
|
-
|
|
175
|
-
let value = serde_json::from_str::<serde_json::Value>(&json_str).ok()?;
|
|
176
|
-
let graph = value.get("@graph")?;
|
|
177
|
-
let items = graph.as_array()?;
|
|
178
|
-
items.iter().find_map(|item| {
|
|
179
|
-
item.get("@type")
|
|
180
|
-
.and_then(|t| t.as_str().map(std::string::ToString::to_string))
|
|
181
|
-
})
|
|
182
|
-
});
|
|
183
|
-
|
|
184
|
-
result.push(StructuredData {
|
|
185
|
-
data_type: StructuredDataType::JsonLd,
|
|
186
|
-
raw_json: json_str,
|
|
187
|
-
schema_type,
|
|
188
|
-
});
|
|
189
|
-
}
|
|
190
|
-
|
|
191
|
-
result
|
|
192
|
-
}
|
|
193
|
-
|
|
194
|
-
/// Scan for @type in JSON string without full parsing.
|
|
195
|
-
fn scan_schema_type(json_str: &str) -> Option<String> {
|
|
196
|
-
let needle = "\"@type\"";
|
|
197
|
-
let start = json_str.find(needle)? + needle.len();
|
|
198
|
-
let bytes = json_str.as_bytes();
|
|
199
|
-
let mut i = start;
|
|
200
|
-
|
|
201
|
-
while i < bytes.len() && bytes[i].is_ascii_whitespace() {
|
|
202
|
-
i += 1;
|
|
203
|
-
}
|
|
204
|
-
if i >= bytes.len() || bytes[i] != b':' {
|
|
205
|
-
return None;
|
|
206
|
-
}
|
|
207
|
-
i += 1;
|
|
208
|
-
while i < bytes.len() && bytes[i].is_ascii_whitespace() {
|
|
209
|
-
i += 1;
|
|
210
|
-
}
|
|
211
|
-
if i >= bytes.len() {
|
|
212
|
-
return None;
|
|
213
|
-
}
|
|
214
|
-
|
|
215
|
-
if bytes[i] == b'[' {
|
|
216
|
-
i += 1;
|
|
217
|
-
while i < bytes.len() && bytes[i].is_ascii_whitespace() {
|
|
218
|
-
i += 1;
|
|
219
|
-
}
|
|
220
|
-
if i >= bytes.len() || bytes[i] != b'"' {
|
|
221
|
-
return None;
|
|
222
|
-
}
|
|
223
|
-
} else if bytes[i] != b'"' {
|
|
224
|
-
return None;
|
|
225
|
-
}
|
|
226
|
-
|
|
227
|
-
let start_quote = i;
|
|
228
|
-
i += 1;
|
|
229
|
-
let mut escaped = false;
|
|
230
|
-
while i < bytes.len() {
|
|
231
|
-
let byte = bytes[i];
|
|
232
|
-
if escaped {
|
|
233
|
-
escaped = false;
|
|
234
|
-
i += 1;
|
|
235
|
-
continue;
|
|
236
|
-
}
|
|
237
|
-
if byte == b'\\' {
|
|
238
|
-
escaped = true;
|
|
239
|
-
i += 1;
|
|
240
|
-
continue;
|
|
241
|
-
}
|
|
242
|
-
if byte == b'"' {
|
|
243
|
-
let end_quote = i;
|
|
244
|
-
let slice = &json_str[start_quote..=end_quote];
|
|
245
|
-
return serde_json::from_str::<String>(slice).ok();
|
|
246
|
-
}
|
|
247
|
-
i += 1;
|
|
248
|
-
}
|
|
249
|
-
|
|
250
|
-
None
|
|
251
|
-
}
|
|
252
|
-
|
|
253
|
-
#[cfg(test)]
|
|
254
|
-
mod tests {
|
|
255
|
-
use super::*;
|
|
256
|
-
|
|
257
|
-
#[test]
|
|
258
|
-
fn test_extract_document_metadata() {
|
|
259
|
-
let mut head_metadata = BTreeMap::new();
|
|
260
|
-
head_metadata.insert("title".to_string(), "Test Title".to_string());
|
|
261
|
-
head_metadata.insert("description".to_string(), "Test Description".to_string());
|
|
262
|
-
head_metadata.insert("keywords".to_string(), "rust, testing".to_string());
|
|
263
|
-
|
|
264
|
-
let doc = extract_document_metadata(head_metadata, Some("en".to_string()), Some("ltr".to_string()));
|
|
265
|
-
|
|
266
|
-
assert_eq!(doc.title, Some("Test Title".to_string()));
|
|
267
|
-
assert_eq!(doc.description, Some("Test Description".to_string()));
|
|
268
|
-
assert_eq!(doc.keywords, vec!["rust", "testing"]);
|
|
269
|
-
assert_eq!(doc.language, Some("en".to_string()));
|
|
270
|
-
}
|
|
271
|
-
|
|
272
|
-
#[test]
|
|
273
|
-
fn test_extract_document_metadata_with_og() {
|
|
274
|
-
let mut head_metadata = BTreeMap::new();
|
|
275
|
-
head_metadata.insert("og-title".to_string(), "OG Title".to_string());
|
|
276
|
-
head_metadata.insert("og-description".to_string(), "OG Description".to_string());
|
|
277
|
-
|
|
278
|
-
let doc = extract_document_metadata(head_metadata, None, None);
|
|
279
|
-
|
|
280
|
-
assert_eq!(doc.open_graph.get("title"), Some(&"OG Title".to_string()));
|
|
281
|
-
assert_eq!(doc.open_graph.get("description"), Some(&"OG Description".to_string()));
|
|
282
|
-
}
|
|
283
|
-
|
|
284
|
-
#[test]
|
|
285
|
-
fn test_keywords_case_insensitive() {
|
|
286
|
-
let mut head_metadata = BTreeMap::new();
|
|
287
|
-
head_metadata.insert("meta-Keywords".to_string(), "Rust, HTML, Markdown".to_string());
|
|
288
|
-
|
|
289
|
-
let doc = extract_document_metadata(head_metadata, None, None);
|
|
290
|
-
assert_eq!(doc.keywords, vec!["Rust", "HTML", "Markdown"]);
|
|
291
|
-
}
|
|
292
|
-
|
|
293
|
-
#[test]
|
|
294
|
-
fn test_keywords_dc_subject() {
|
|
295
|
-
let mut head_metadata = BTreeMap::new();
|
|
296
|
-
head_metadata.insert("meta-DC.subject".to_string(), "weather, forecast".to_string());
|
|
297
|
-
|
|
298
|
-
let doc = extract_document_metadata(head_metadata, None, None);
|
|
299
|
-
assert_eq!(doc.keywords, vec!["weather", "forecast"]);
|
|
300
|
-
}
|
|
301
|
-
|
|
302
|
-
#[test]
|
|
303
|
-
fn test_keywords_dc_keywords() {
|
|
304
|
-
let mut head_metadata = BTreeMap::new();
|
|
305
|
-
head_metadata.insert("meta-DC.keywords".to_string(), "climate, data".to_string());
|
|
306
|
-
|
|
307
|
-
let doc = extract_document_metadata(head_metadata, None, None);
|
|
308
|
-
assert_eq!(doc.keywords, vec!["climate", "data"]);
|
|
309
|
-
}
|
|
310
|
-
|
|
311
|
-
#[test]
|
|
312
|
-
fn test_keywords_dcterms_subject() {
|
|
313
|
-
let mut head_metadata = BTreeMap::new();
|
|
314
|
-
head_metadata.insert("meta-DCTERMS.subject".to_string(), "science, research".to_string());
|
|
315
|
-
|
|
316
|
-
let doc = extract_document_metadata(head_metadata, None, None);
|
|
317
|
-
assert_eq!(doc.keywords, vec!["science", "research"]);
|
|
318
|
-
}
|
|
319
|
-
|
|
320
|
-
#[test]
|
|
321
|
-
fn test_keywords_news_keywords() {
|
|
322
|
-
let mut head_metadata = BTreeMap::new();
|
|
323
|
-
head_metadata.insert("meta-news_keywords".to_string(), "breaking, world".to_string());
|
|
324
|
-
|
|
325
|
-
let doc = extract_document_metadata(head_metadata, None, None);
|
|
326
|
-
assert_eq!(doc.keywords, vec!["breaking", "world"]);
|
|
327
|
-
}
|
|
328
|
-
|
|
329
|
-
#[test]
|
|
330
|
-
fn test_keywords_citation_keywords() {
|
|
331
|
-
let mut head_metadata = BTreeMap::new();
|
|
332
|
-
head_metadata.insert("meta-citation_keywords".to_string(), "biology, genetics".to_string());
|
|
333
|
-
|
|
334
|
-
let doc = extract_document_metadata(head_metadata, None, None);
|
|
335
|
-
assert_eq!(doc.keywords, vec!["biology", "genetics"]);
|
|
336
|
-
}
|
|
337
|
-
|
|
338
|
-
#[test]
|
|
339
|
-
fn test_dc_title_and_description_fallback() {
|
|
340
|
-
let mut head_metadata = BTreeMap::new();
|
|
341
|
-
head_metadata.insert("meta-DC.title".to_string(), "DC Title".to_string());
|
|
342
|
-
head_metadata.insert("meta-DC.description".to_string(), "DC Description".to_string());
|
|
343
|
-
head_metadata.insert("meta-DC.creator".to_string(), "DC Author".to_string());
|
|
344
|
-
|
|
345
|
-
let doc = extract_document_metadata(head_metadata, None, None);
|
|
346
|
-
assert_eq!(doc.title, Some("DC Title".to_string()));
|
|
347
|
-
assert_eq!(doc.description, Some("DC Description".to_string()));
|
|
348
|
-
assert_eq!(doc.author, Some("DC Author".to_string()));
|
|
349
|
-
}
|
|
350
|
-
|
|
351
|
-
#[test]
|
|
352
|
-
fn test_dc_does_not_override_standard_fields() {
|
|
353
|
-
let mut head_metadata = BTreeMap::new();
|
|
354
|
-
// Standard fields come first alphabetically in BTreeMap
|
|
355
|
-
head_metadata.insert("description".to_string(), "Standard Description".to_string());
|
|
356
|
-
head_metadata.insert("meta-DC.description".to_string(), "DC Description".to_string());
|
|
357
|
-
head_metadata.insert("title".to_string(), "Standard Title".to_string());
|
|
358
|
-
head_metadata.insert("meta-DC.title".to_string(), "DC Title".to_string());
|
|
359
|
-
|
|
360
|
-
let doc = extract_document_metadata(head_metadata, None, None);
|
|
361
|
-
assert_eq!(doc.title, Some("Standard Title".to_string()));
|
|
362
|
-
assert_eq!(doc.description, Some("Standard Description".to_string()));
|
|
363
|
-
}
|
|
364
|
-
|
|
365
|
-
#[test]
|
|
366
|
-
fn test_case_insensitive_title_description_author() {
|
|
367
|
-
let mut head_metadata = BTreeMap::new();
|
|
368
|
-
head_metadata.insert("meta-Title".to_string(), "My Title".to_string());
|
|
369
|
-
head_metadata.insert("meta-Description".to_string(), "My Desc".to_string());
|
|
370
|
-
head_metadata.insert("meta-Author".to_string(), "My Author".to_string());
|
|
371
|
-
|
|
372
|
-
let doc = extract_document_metadata(head_metadata, None, None);
|
|
373
|
-
assert_eq!(doc.title, Some("My Title".to_string()));
|
|
374
|
-
assert_eq!(doc.description, Some("My Desc".to_string()));
|
|
375
|
-
assert_eq!(doc.author, Some("My Author".to_string()));
|
|
376
|
-
}
|
|
377
|
-
|
|
378
|
-
#[test]
|
|
379
|
-
fn test_dcterms_remaining_go_to_meta_tags() {
|
|
380
|
-
let mut head_metadata = BTreeMap::new();
|
|
381
|
-
head_metadata.insert("meta-DCTERMS.license".to_string(), "MIT".to_string());
|
|
382
|
-
|
|
383
|
-
let doc = extract_document_metadata(head_metadata, None, None);
|
|
384
|
-
assert_eq!(doc.meta_tags.get("dcterms_license"), Some(&"MIT".to_string()));
|
|
385
|
-
}
|
|
386
|
-
|
|
387
|
-
#[test]
|
|
388
|
-
fn test_scan_schema_type() {
|
|
389
|
-
let json = r#"{"@type":"Article","title":"Test"}"#;
|
|
390
|
-
assert_eq!(scan_schema_type(json), Some("Article".to_string()));
|
|
391
|
-
|
|
392
|
-
let json_array = r#"{"@type":["Article","NewsArticle"]}"#;
|
|
393
|
-
assert_eq!(scan_schema_type(json_array), Some("Article".to_string()));
|
|
394
|
-
|
|
395
|
-
let no_type = r#"{"title":"Test"}"#;
|
|
396
|
-
assert_eq!(scan_schema_type(no_type), None);
|
|
397
|
-
}
|
|
398
|
-
}
|
|
@@ -1,288 +0,0 @@
|
|
|
1
|
-
#![allow(clippy::cast_precision_loss, clippy::cast_sign_loss, clippy::unused_self)]
|
|
2
|
-
//! Metadata extraction for HTML to Markdown conversion.
|
|
3
|
-
//!
|
|
4
|
-
//! This module provides comprehensive, type-safe metadata extraction during HTML-to-Markdown
|
|
5
|
-
//! conversion, enabling content analysis, SEO optimization, and document indexing workflows.
|
|
6
|
-
//! Metadata includes:
|
|
7
|
-
//! - **Document metadata**: Title, description, author, language, canonical URL, Open Graph, Twitter Card
|
|
8
|
-
//! - **Headers**: Heading elements (h1-h6) with hierarchy, IDs, and positions
|
|
9
|
-
//! - **Links**: Hyperlinks with type classification (anchor, internal, external, email, phone)
|
|
10
|
-
//! - **Images**: Image elements with source, alt text, dimensions, and type (data URI, external, etc.)
|
|
11
|
-
//! - **Structured data**: JSON-LD, Microdata, and RDFa blocks
|
|
12
|
-
//!
|
|
13
|
-
//! The implementation follows a single-pass collector pattern for zero-overhead extraction
|
|
14
|
-
//! when metadata features are disabled.
|
|
15
|
-
//!
|
|
16
|
-
//! # Architecture
|
|
17
|
-
//!
|
|
18
|
-
//! Metadata extraction uses the [`MetadataCollector`] pattern:
|
|
19
|
-
//! - **Single-pass collection**: Metadata is gathered during the primary tree traversal without additional passes
|
|
20
|
-
//! - **Zero overhead when disabled**: Entire module can be compiled out via feature flags
|
|
21
|
-
//! - **Configurable granularity**: Use [`MetadataConfig`] to select which metadata types to extract
|
|
22
|
-
//! - **Type-safe APIs**: All metadata types are enum-based with exhaustive matching
|
|
23
|
-
//! - **Memory-bounded**: Size limits prevent memory exhaustion from adversarial documents
|
|
24
|
-
//! - **Pre-allocated buffers**: Typical documents (32 headers, 64 links, 16 images) handled efficiently
|
|
25
|
-
//!
|
|
26
|
-
//! # Type Overview
|
|
27
|
-
//!
|
|
28
|
-
//! ## Enumerations
|
|
29
|
-
//!
|
|
30
|
-
//! - [`TextDirection`]: Document directionality (LTR, RTL, Auto)
|
|
31
|
-
//! - [`LinkType`]: Link classification (Anchor, Internal, External, Email, Phone, Other)
|
|
32
|
-
//! - [`ImageType`]: Image source type (DataUri, External, Relative, InlineSvg)
|
|
33
|
-
//! - [`StructuredDataType`]: Structured data format (JsonLd, Microdata, RDFa)
|
|
34
|
-
//!
|
|
35
|
-
//! ## Structures
|
|
36
|
-
//!
|
|
37
|
-
//! - [`DocumentMetadata`]: Head-level metadata with maps for Open Graph and Twitter Card
|
|
38
|
-
//! - [`HeaderMetadata`]: Heading element with level (1-6), text, ID, hierarchy depth, and position
|
|
39
|
-
//! - [`LinkMetadata`]: Hyperlink with href, text, title, type, rel attributes, and custom attributes
|
|
40
|
-
//! - [`ImageMetadata`]: Image element with src, alt, title, dimensions, type, and attributes
|
|
41
|
-
//! - [`StructuredData`]: Structured data block with type and raw JSON
|
|
42
|
-
//! - [`MetadataConfig`]: Configuration controlling extraction granularity and size limits
|
|
43
|
-
//! - [`HtmlMetadata`]: Top-level result containing all extracted metadata
|
|
44
|
-
//!
|
|
45
|
-
//! # Examples
|
|
46
|
-
//!
|
|
47
|
-
//! ## Basic Usage with `convert()`
|
|
48
|
-
//!
|
|
49
|
-
//! ```text
|
|
50
|
-
//! use html_to_markdown_rs::convert;
|
|
51
|
-
//!
|
|
52
|
-
//! let html = r#"
|
|
53
|
-
//! <html lang="en">
|
|
54
|
-
//! <head>
|
|
55
|
-
//! <title>My Article</title>
|
|
56
|
-
//! <meta name="description" content="An interesting read">
|
|
57
|
-
//! </head>
|
|
58
|
-
//! <body>
|
|
59
|
-
//! <h1 id="main">Title</h1>
|
|
60
|
-
//! <a href="https://example.com">External Link</a>
|
|
61
|
-
//! <img src="photo.jpg" alt="A photo">
|
|
62
|
-
//! </body>
|
|
63
|
-
//! </html>
|
|
64
|
-
//! "#;
|
|
65
|
-
//!
|
|
66
|
-
//! let result = convert(html, None)?;
|
|
67
|
-
//! let metadata = result.metadata.unwrap();
|
|
68
|
-
//!
|
|
69
|
-
//! // Access document metadata
|
|
70
|
-
//! assert_eq!(metadata.document.title, Some("My Article".to_string()));
|
|
71
|
-
//! assert_eq!(metadata.document.language, Some("en".to_string()));
|
|
72
|
-
//!
|
|
73
|
-
//! // Access headers
|
|
74
|
-
//! assert_eq!(metadata.headers.len(), 1);
|
|
75
|
-
//! assert_eq!(metadata.headers[0].level, 1);
|
|
76
|
-
//! assert_eq!(metadata.headers[0].id, Some("main".to_string()));
|
|
77
|
-
//!
|
|
78
|
-
//! // Access links
|
|
79
|
-
//! assert_eq!(metadata.links.len(), 1);
|
|
80
|
-
//! assert_eq!(metadata.links[0].link_type, LinkType::External);
|
|
81
|
-
//!
|
|
82
|
-
//! // Access images
|
|
83
|
-
//! assert_eq!(metadata.images.len(), 1);
|
|
84
|
-
//! assert_eq!(metadata.images[0].image_type, ImageType::Relative);
|
|
85
|
-
//! # Ok::<(), html_to_markdown_rs::ConversionError>(())
|
|
86
|
-
//! ```
|
|
87
|
-
//!
|
|
88
|
-
//! ## Selective Extraction
|
|
89
|
-
//!
|
|
90
|
-
//! ```text
|
|
91
|
-
//! use html_to_markdown_rs::{convert, ConversionOptions};
|
|
92
|
-
//!
|
|
93
|
-
//! let options = ConversionOptions {
|
|
94
|
-
//! extract_metadata: false, // Disable metadata extraction
|
|
95
|
-
//! ..Default::default()
|
|
96
|
-
//! };
|
|
97
|
-
//!
|
|
98
|
-
//! let result = convert(html, Some(options))?;
|
|
99
|
-
//! assert!(result.metadata.is_none()); // Metadata not extracted
|
|
100
|
-
//! # Ok::<(), html_to_markdown_rs::ConversionError>(())
|
|
101
|
-
//! ```
|
|
102
|
-
//!
|
|
103
|
-
//! ## Analyzing Link Types
|
|
104
|
-
//!
|
|
105
|
-
//! ```text
|
|
106
|
-
//! use html_to_markdown_rs::convert;
|
|
107
|
-
//! use html_to_markdown_rs::metadata::LinkType;
|
|
108
|
-
//!
|
|
109
|
-
//! let result = convert(html, None)?;
|
|
110
|
-
//! let metadata = result.metadata.unwrap();
|
|
111
|
-
//!
|
|
112
|
-
//! for link in &metadata.links {
|
|
113
|
-
//! match link.link_type {
|
|
114
|
-
//! LinkType::External => println!("External: {}", link.href),
|
|
115
|
-
//! LinkType::Internal => println!("Internal: {}", link.href),
|
|
116
|
-
//! LinkType::Anchor => println!("Anchor: {}", link.href),
|
|
117
|
-
//! LinkType::Email => println!("Email: {}", link.href),
|
|
118
|
-
//! _ => {}
|
|
119
|
-
//! }
|
|
120
|
-
//! }
|
|
121
|
-
//! # Ok::<(), html_to_markdown_rs::ConversionError>(())
|
|
122
|
-
//! ```
|
|
123
|
-
//!
|
|
124
|
-
//! # Serialization
|
|
125
|
-
//!
|
|
126
|
-
//! All types in this module support serialization via `serde` when the `metadata` feature is enabled.
|
|
127
|
-
//! This enables easy export to JSON, YAML, or other formats:
|
|
128
|
-
//!
|
|
129
|
-
//! ```text
|
|
130
|
-
//! use html_to_markdown_rs::convert;
|
|
131
|
-
//!
|
|
132
|
-
//! let result = convert(html, None)?;
|
|
133
|
-
//! if let Some(metadata) = &result.metadata {
|
|
134
|
-
//! let json = serde_json::to_string_pretty(metadata)?;
|
|
135
|
-
//! println!("{}", json);
|
|
136
|
-
//! }
|
|
137
|
-
//! # Ok::<(), Box<dyn std::error::Error>>(())
|
|
138
|
-
//! ```
|
|
139
|
-
|
|
140
|
-
pub mod collector;
|
|
141
|
-
pub mod config;
|
|
142
|
-
pub mod extraction;
|
|
143
|
-
pub mod types;
|
|
144
|
-
|
|
145
|
-
// Re-export public types
|
|
146
|
-
pub use collector::MetadataCollector;
|
|
147
|
-
pub use config::{DEFAULT_MAX_STRUCTURED_DATA_SIZE, MetadataConfig, MetadataConfigUpdate};
|
|
148
|
-
pub use types::{
|
|
149
|
-
DocumentMetadata, HeaderMetadata, HtmlMetadata, ImageMetadata, ImageType, LinkMetadata, LinkType, StructuredData,
|
|
150
|
-
StructuredDataType, TextDirection,
|
|
151
|
-
};
|
|
152
|
-
|
|
153
|
-
// Internal handle type for shared mutable access during tree traversal
|
|
154
|
-
use std::cell::RefCell;
|
|
155
|
-
use std::rc::Rc;
|
|
156
|
-
|
|
157
|
-
/// Handle to a metadata collector via reference-counted mutable cell.
|
|
158
|
-
///
|
|
159
|
-
/// Used internally for sharing collector state across the tree traversal.
|
|
160
|
-
///
|
|
161
|
-
/// # Examples
|
|
162
|
-
///
|
|
163
|
-
/// ```text
|
|
164
|
-
/// let collector = MetadataCollector::new(MetadataConfig::default());
|
|
165
|
-
/// let handle = Rc::new(RefCell::new(collector));
|
|
166
|
-
///
|
|
167
|
-
/// // In tree walk, can be passed and borrowed
|
|
168
|
-
/// handle.borrow_mut().add_header(1, "Title".to_string(), None, 0, 100);
|
|
169
|
-
///
|
|
170
|
-
/// let metadata = handle.take().finish();
|
|
171
|
-
/// ```
|
|
172
|
-
#[allow(dead_code)]
|
|
173
|
-
pub(crate) type MetadataCollectorHandle = Rc<RefCell<MetadataCollector>>;
|
|
174
|
-
|
|
175
|
-
#[cfg(test)]
|
|
176
|
-
mod tests {
|
|
177
|
-
use super::*;
|
|
178
|
-
|
|
179
|
-
#[test]
|
|
180
|
-
fn test_text_direction_parse() {
|
|
181
|
-
assert_eq!(TextDirection::parse("ltr"), Some(TextDirection::LeftToRight));
|
|
182
|
-
assert_eq!(TextDirection::parse("rtl"), Some(TextDirection::RightToLeft));
|
|
183
|
-
assert_eq!(TextDirection::parse("auto"), Some(TextDirection::Auto));
|
|
184
|
-
assert_eq!(TextDirection::parse("invalid"), None);
|
|
185
|
-
assert_eq!(TextDirection::parse("LTR"), Some(TextDirection::LeftToRight));
|
|
186
|
-
}
|
|
187
|
-
|
|
188
|
-
#[test]
|
|
189
|
-
fn test_text_direction_display() {
|
|
190
|
-
assert_eq!(TextDirection::LeftToRight.to_string(), "ltr");
|
|
191
|
-
assert_eq!(TextDirection::RightToLeft.to_string(), "rtl");
|
|
192
|
-
assert_eq!(TextDirection::Auto.to_string(), "auto");
|
|
193
|
-
}
|
|
194
|
-
|
|
195
|
-
#[test]
|
|
196
|
-
fn test_link_classification() {
|
|
197
|
-
assert_eq!(LinkMetadata::classify_link("#section"), LinkType::Anchor);
|
|
198
|
-
assert_eq!(LinkMetadata::classify_link("mailto:test@example.com"), LinkType::Email);
|
|
199
|
-
assert_eq!(LinkMetadata::classify_link("tel:+1234567890"), LinkType::Phone);
|
|
200
|
-
assert_eq!(LinkMetadata::classify_link("https://example.com"), LinkType::External);
|
|
201
|
-
assert_eq!(LinkMetadata::classify_link("http://example.com"), LinkType::External);
|
|
202
|
-
assert_eq!(LinkMetadata::classify_link("/path/to/page"), LinkType::Internal);
|
|
203
|
-
assert_eq!(LinkMetadata::classify_link("../relative"), LinkType::Internal);
|
|
204
|
-
assert_eq!(LinkMetadata::classify_link("./same"), LinkType::Internal);
|
|
205
|
-
}
|
|
206
|
-
|
|
207
|
-
#[test]
|
|
208
|
-
fn test_header_validation() {
|
|
209
|
-
let valid = HeaderMetadata {
|
|
210
|
-
level: 3,
|
|
211
|
-
text: "Title".to_string(),
|
|
212
|
-
id: None,
|
|
213
|
-
depth: 2,
|
|
214
|
-
html_offset: 100,
|
|
215
|
-
};
|
|
216
|
-
assert!(valid.is_valid());
|
|
217
|
-
|
|
218
|
-
let invalid_high = HeaderMetadata {
|
|
219
|
-
level: 7,
|
|
220
|
-
text: "Title".to_string(),
|
|
221
|
-
id: None,
|
|
222
|
-
depth: 2,
|
|
223
|
-
html_offset: 100,
|
|
224
|
-
};
|
|
225
|
-
assert!(!invalid_high.is_valid());
|
|
226
|
-
|
|
227
|
-
let invalid_low = HeaderMetadata {
|
|
228
|
-
level: 0,
|
|
229
|
-
text: "Title".to_string(),
|
|
230
|
-
id: None,
|
|
231
|
-
depth: 2,
|
|
232
|
-
html_offset: 100,
|
|
233
|
-
};
|
|
234
|
-
assert!(!invalid_low.is_valid());
|
|
235
|
-
}
|
|
236
|
-
|
|
237
|
-
#[test]
|
|
238
|
-
fn test_document_metadata_default() {
|
|
239
|
-
let doc = DocumentMetadata::default();
|
|
240
|
-
|
|
241
|
-
assert!(doc.title.is_none());
|
|
242
|
-
assert!(doc.description.is_none());
|
|
243
|
-
assert!(doc.keywords.is_empty());
|
|
244
|
-
assert!(doc.open_graph.is_empty());
|
|
245
|
-
assert!(doc.twitter_card.is_empty());
|
|
246
|
-
assert!(doc.meta_tags.is_empty());
|
|
247
|
-
}
|
|
248
|
-
|
|
249
|
-
#[test]
|
|
250
|
-
fn test_image_type_classification() {
|
|
251
|
-
let data_uri = ImageMetadata {
|
|
252
|
-
src: "data:image/png;base64,iVBORw0KG...".to_string(),
|
|
253
|
-
alt: None,
|
|
254
|
-
title: None,
|
|
255
|
-
dimensions: None,
|
|
256
|
-
image_type: ImageType::DataUri,
|
|
257
|
-
attributes: Default::default(),
|
|
258
|
-
};
|
|
259
|
-
assert_eq!(data_uri.image_type, ImageType::DataUri);
|
|
260
|
-
|
|
261
|
-
let external = ImageMetadata {
|
|
262
|
-
src: "https://example.com/image.jpg".to_string(),
|
|
263
|
-
alt: None,
|
|
264
|
-
title: None,
|
|
265
|
-
dimensions: None,
|
|
266
|
-
image_type: ImageType::External,
|
|
267
|
-
attributes: Default::default(),
|
|
268
|
-
};
|
|
269
|
-
assert_eq!(external.image_type, ImageType::External);
|
|
270
|
-
}
|
|
271
|
-
|
|
272
|
-
#[test]
|
|
273
|
-
fn test_link_type_display() {
|
|
274
|
-
assert_eq!(LinkType::Anchor.to_string(), "anchor");
|
|
275
|
-
assert_eq!(LinkType::Internal.to_string(), "internal");
|
|
276
|
-
assert_eq!(LinkType::External.to_string(), "external");
|
|
277
|
-
assert_eq!(LinkType::Email.to_string(), "email");
|
|
278
|
-
assert_eq!(LinkType::Phone.to_string(), "phone");
|
|
279
|
-
assert_eq!(LinkType::Other.to_string(), "other");
|
|
280
|
-
}
|
|
281
|
-
|
|
282
|
-
#[test]
|
|
283
|
-
fn test_structured_data_type_display() {
|
|
284
|
-
assert_eq!(StructuredDataType::JsonLd.to_string(), "json_ld");
|
|
285
|
-
assert_eq!(StructuredDataType::Microdata.to_string(), "microdata");
|
|
286
|
-
assert_eq!(StructuredDataType::RDFa.to_string(), "rdfa");
|
|
287
|
-
}
|
|
288
|
-
}
|