html-to-markdown 3.4.0 → 3.6.0.pre.rc.23

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (188) hide show
  1. checksums.yaml +4 -4
  2. data/LICENSE +21 -0
  3. data/README.md +347 -0
  4. data/Steepfile +10 -2
  5. data/ext/html_to_markdown_rb/Cargo.toml +3 -2
  6. data/ext/html_to_markdown_rb/extconf.rb +5 -5
  7. data/ext/html_to_markdown_rb/native/Cargo.lock +962 -0
  8. data/ext/html_to_markdown_rb/native/Cargo.toml +6 -11
  9. data/ext/html_to_markdown_rb/native/extconf.rb +14 -0
  10. data/ext/html_to_markdown_rb/src/lib.rs +1715 -646
  11. data/lib/html_to_markdown/native.rb +913 -37
  12. data/lib/html_to_markdown/version.rb +3 -3
  13. data/lib/html_to_markdown.rb +9 -4
  14. data/lib/html_to_markdown_rb.so +0 -0
  15. data/sig/types.rbs +59 -292
  16. metadata +32 -179
  17. data/ext/html_to_markdown_rb/Makefile +0 -592
  18. data/lib/bin/html-to-markdown +0 -0
  19. data/vendor/Cargo.toml +0 -33
  20. data/vendor/html-to-markdown-rs/Cargo.toml +0 -54
  21. data/vendor/html-to-markdown-rs/README.md +0 -278
  22. data/vendor/html-to-markdown-rs/examples/basic.rs +0 -24
  23. data/vendor/html-to-markdown-rs/examples/table.rs +0 -25
  24. data/vendor/html-to-markdown-rs/examples/test_deser.rs +0 -12
  25. data/vendor/html-to-markdown-rs/examples/test_escape.rs +0 -58
  26. data/vendor/html-to-markdown-rs/examples/test_inline_formatting.rs +0 -113
  27. data/vendor/html-to-markdown-rs/examples/test_lists.rs +0 -39
  28. data/vendor/html-to-markdown-rs/examples/test_semantic_tags.rs +0 -89
  29. data/vendor/html-to-markdown-rs/examples/test_tables.rs +0 -100
  30. data/vendor/html-to-markdown-rs/examples/test_task_lists.rs +0 -61
  31. data/vendor/html-to-markdown-rs/examples/test_whitespace.rs +0 -34
  32. data/vendor/html-to-markdown-rs/src/convert_api.rs +0 -349
  33. data/vendor/html-to-markdown-rs/src/converter/block/blockquote.rs +0 -178
  34. data/vendor/html-to-markdown-rs/src/converter/block/container.rs +0 -114
  35. data/vendor/html-to-markdown-rs/src/converter/block/div.rs +0 -149
  36. data/vendor/html-to-markdown-rs/src/converter/block/heading.rs +0 -428
  37. data/vendor/html-to-markdown-rs/src/converter/block/horizontal_rule.rs +0 -103
  38. data/vendor/html-to-markdown-rs/src/converter/block/line_break.rs +0 -89
  39. data/vendor/html-to-markdown-rs/src/converter/block/mod.rs +0 -10
  40. data/vendor/html-to-markdown-rs/src/converter/block/paragraph.rs +0 -140
  41. data/vendor/html-to-markdown-rs/src/converter/block/preformatted.rs +0 -298
  42. data/vendor/html-to-markdown-rs/src/converter/block/table/builder.rs +0 -453
  43. data/vendor/html-to-markdown-rs/src/converter/block/table/caption.rs +0 -44
  44. data/vendor/html-to-markdown-rs/src/converter/block/table/cell.rs +0 -276
  45. data/vendor/html-to-markdown-rs/src/converter/block/table/cells.rs +0 -336
  46. data/vendor/html-to-markdown-rs/src/converter/block/table/layout.rs +0 -58
  47. data/vendor/html-to-markdown-rs/src/converter/block/table/mod.rs +0 -266
  48. data/vendor/html-to-markdown-rs/src/converter/block/table/scanner.rs +0 -146
  49. data/vendor/html-to-markdown-rs/src/converter/block/table/utils.rs +0 -34
  50. data/vendor/html-to-markdown-rs/src/converter/block/unknown.rs +0 -138
  51. data/vendor/html-to-markdown-rs/src/converter/context.rs +0 -208
  52. data/vendor/html-to-markdown-rs/src/converter/dom_context.rs +0 -337
  53. data/vendor/html-to-markdown-rs/src/converter/form/elements.rs +0 -770
  54. data/vendor/html-to-markdown-rs/src/converter/form/mod.rs +0 -82
  55. data/vendor/html-to-markdown-rs/src/converter/format/djot.rs +0 -64
  56. data/vendor/html-to-markdown-rs/src/converter/format/markdown.rs +0 -59
  57. data/vendor/html-to-markdown-rs/src/converter/format/mod.rs +0 -43
  58. data/vendor/html-to-markdown-rs/src/converter/handlers/blockquote.rs +0 -173
  59. data/vendor/html-to-markdown-rs/src/converter/handlers/code_block.rs +0 -434
  60. data/vendor/html-to-markdown-rs/src/converter/handlers/graphic.rs +0 -234
  61. data/vendor/html-to-markdown-rs/src/converter/handlers/image.rs +0 -282
  62. data/vendor/html-to-markdown-rs/src/converter/handlers/link.rs +0 -316
  63. data/vendor/html-to-markdown-rs/src/converter/handlers/mod.rs +0 -26
  64. data/vendor/html-to-markdown-rs/src/converter/inline/code.rs +0 -306
  65. data/vendor/html-to-markdown-rs/src/converter/inline/emphasis.rs +0 -345
  66. data/vendor/html-to-markdown-rs/src/converter/inline/link.rs +0 -428
  67. data/vendor/html-to-markdown-rs/src/converter/inline/mod.rs +0 -237
  68. data/vendor/html-to-markdown-rs/src/converter/inline/ruby.rs +0 -337
  69. data/vendor/html-to-markdown-rs/src/converter/inline/semantic/marks.rs +0 -566
  70. data/vendor/html-to-markdown-rs/src/converter/inline/semantic/mod.rs +0 -86
  71. data/vendor/html-to-markdown-rs/src/converter/inline/semantic/typography.rs +0 -558
  72. data/vendor/html-to-markdown-rs/src/converter/list/definition.rs +0 -232
  73. data/vendor/html-to-markdown-rs/src/converter/list/item.rs +0 -332
  74. data/vendor/html-to-markdown-rs/src/converter/list/mod.rs +0 -70
  75. data/vendor/html-to-markdown-rs/src/converter/list/ordered.rs +0 -201
  76. data/vendor/html-to-markdown-rs/src/converter/list/unordered.rs +0 -195
  77. data/vendor/html-to-markdown-rs/src/converter/list/utils.rs +0 -314
  78. data/vendor/html-to-markdown-rs/src/converter/main.rs +0 -710
  79. data/vendor/html-to-markdown-rs/src/converter/main_helpers.rs +0 -452
  80. data/vendor/html-to-markdown-rs/src/converter/media/embedded.rs +0 -393
  81. data/vendor/html-to-markdown-rs/src/converter/media/graphic.rs +0 -4
  82. data/vendor/html-to-markdown-rs/src/converter/media/image.rs +0 -183
  83. data/vendor/html-to-markdown-rs/src/converter/media/mod.rs +0 -87
  84. data/vendor/html-to-markdown-rs/src/converter/media/svg.rs +0 -280
  85. data/vendor/html-to-markdown-rs/src/converter/metadata.rs +0 -220
  86. data/vendor/html-to-markdown-rs/src/converter/mod.rs +0 -156
  87. data/vendor/html-to-markdown-rs/src/converter/plain_text.rs +0 -516
  88. data/vendor/html-to-markdown-rs/src/converter/preprocessing_helpers.rs +0 -201
  89. data/vendor/html-to-markdown-rs/src/converter/reference_collector.rs +0 -69
  90. data/vendor/html-to-markdown-rs/src/converter/semantic/attributes.rs +0 -269
  91. data/vendor/html-to-markdown-rs/src/converter/semantic/definition_list.rs +0 -266
  92. data/vendor/html-to-markdown-rs/src/converter/semantic/figure.rs +0 -391
  93. data/vendor/html-to-markdown-rs/src/converter/semantic/mod.rs +0 -112
  94. data/vendor/html-to-markdown-rs/src/converter/semantic/sectioning.rs +0 -85
  95. data/vendor/html-to-markdown-rs/src/converter/semantic/summary.rs +0 -324
  96. data/vendor/html-to-markdown-rs/src/converter/text/mod.rs +0 -8
  97. data/vendor/html-to-markdown-rs/src/converter/text/processing.rs +0 -56
  98. data/vendor/html-to-markdown-rs/src/converter/text_node.rs +0 -269
  99. data/vendor/html-to-markdown-rs/src/converter/utility/attributes.rs +0 -151
  100. data/vendor/html-to-markdown-rs/src/converter/utility/caching.rs +0 -74
  101. data/vendor/html-to-markdown-rs/src/converter/utility/content.rs +0 -271
  102. data/vendor/html-to-markdown-rs/src/converter/utility/mod.rs +0 -17
  103. data/vendor/html-to-markdown-rs/src/converter/utility/preprocessing.rs +0 -1002
  104. data/vendor/html-to-markdown-rs/src/converter/utility/serialization.rs +0 -126
  105. data/vendor/html-to-markdown-rs/src/converter/utility/siblings.rs +0 -97
  106. data/vendor/html-to-markdown-rs/src/converter/visitor_hooks.rs +0 -189
  107. data/vendor/html-to-markdown-rs/src/error.rs +0 -43
  108. data/vendor/html-to-markdown-rs/src/exports.rs +0 -24
  109. data/vendor/html-to-markdown-rs/src/inline_images.rs +0 -336
  110. data/vendor/html-to-markdown-rs/src/lib.rs +0 -139
  111. data/vendor/html-to-markdown-rs/src/metadata/collector.rs +0 -457
  112. data/vendor/html-to-markdown-rs/src/metadata/config.rs +0 -394
  113. data/vendor/html-to-markdown-rs/src/metadata/extraction.rs +0 -398
  114. data/vendor/html-to-markdown-rs/src/metadata/mod.rs +0 -288
  115. data/vendor/html-to-markdown-rs/src/metadata/types.rs +0 -477
  116. data/vendor/html-to-markdown-rs/src/options/conversion.rs +0 -559
  117. data/vendor/html-to-markdown-rs/src/options/inline_image.rs +0 -111
  118. data/vendor/html-to-markdown-rs/src/options/mod.rs +0 -20
  119. data/vendor/html-to-markdown-rs/src/options/preprocessing.rs +0 -201
  120. data/vendor/html-to-markdown-rs/src/options/validation.rs +0 -416
  121. data/vendor/html-to-markdown-rs/src/prelude.rs +0 -1
  122. data/vendor/html-to-markdown-rs/src/rcdom.rs +0 -487
  123. data/vendor/html-to-markdown-rs/src/text.rs +0 -358
  124. data/vendor/html-to-markdown-rs/src/types/document.rs +0 -191
  125. data/vendor/html-to-markdown-rs/src/types/mod.rs +0 -17
  126. data/vendor/html-to-markdown-rs/src/types/result.rs +0 -54
  127. data/vendor/html-to-markdown-rs/src/types/structure_builder.rs +0 -791
  128. data/vendor/html-to-markdown-rs/src/types/structure_collector.rs +0 -483
  129. data/vendor/html-to-markdown-rs/src/types/tables.rs +0 -52
  130. data/vendor/html-to-markdown-rs/src/types/warnings.rs +0 -33
  131. data/vendor/html-to-markdown-rs/src/validation.rs +0 -158
  132. data/vendor/html-to-markdown-rs/src/visitor/default_impl.rs +0 -63
  133. data/vendor/html-to-markdown-rs/src/visitor/mod.rs +0 -41
  134. data/vendor/html-to-markdown-rs/src/visitor/traits.rs +0 -370
  135. data/vendor/html-to-markdown-rs/src/visitor/types.rs +0 -319
  136. data/vendor/html-to-markdown-rs/src/visitor_helpers/helpers/callbacks/mod.rs +0 -1
  137. data/vendor/html-to-markdown-rs/src/visitor_helpers/helpers/content.rs +0 -126
  138. data/vendor/html-to-markdown-rs/src/visitor_helpers/helpers/mod.rs +0 -27
  139. data/vendor/html-to-markdown-rs/src/visitor_helpers/helpers/state.rs +0 -110
  140. data/vendor/html-to-markdown-rs/src/visitor_helpers/helpers/traversal.rs +0 -250
  141. data/vendor/html-to-markdown-rs/src/visitor_helpers.rs +0 -597
  142. data/vendor/html-to-markdown-rs/src/wrapper/sync.rs +0 -413
  143. data/vendor/html-to-markdown-rs/src/wrapper/utils.rs +0 -290
  144. data/vendor/html-to-markdown-rs/src/wrapper.rs +0 -9
  145. data/vendor/html-to-markdown-rs/tests/br_in_inline_test.rs +0 -87
  146. data/vendor/html-to-markdown-rs/tests/commonmark_compliance_test.rs +0 -297
  147. data/vendor/html-to-markdown-rs/tests/djot_output_test.rs +0 -153
  148. data/vendor/html-to-markdown-rs/tests/exclude_selectors_test.rs +0 -132
  149. data/vendor/html-to-markdown-rs/tests/integration_test.rs +0 -631
  150. data/vendor/html-to-markdown-rs/tests/issue_121_regressions.rs +0 -49
  151. data/vendor/html-to-markdown-rs/tests/issue_127_regressions.rs +0 -58
  152. data/vendor/html-to-markdown-rs/tests/issue_128_regressions.rs +0 -17
  153. data/vendor/html-to-markdown-rs/tests/issue_131_regressions.rs +0 -41
  154. data/vendor/html-to-markdown-rs/tests/issue_134_regressions.rs +0 -40
  155. data/vendor/html-to-markdown-rs/tests/issue_139_regressions.rs +0 -26
  156. data/vendor/html-to-markdown-rs/tests/issue_140_regressions.rs +0 -185
  157. data/vendor/html-to-markdown-rs/tests/issue_143_regressions.rs +0 -100
  158. data/vendor/html-to-markdown-rs/tests/issue_145_regressions.rs +0 -133
  159. data/vendor/html-to-markdown-rs/tests/issue_146_regressions.rs +0 -144
  160. data/vendor/html-to-markdown-rs/tests/issue_176_regressions.rs +0 -62
  161. data/vendor/html-to-markdown-rs/tests/issue_190_regressions.rs +0 -128
  162. data/vendor/html-to-markdown-rs/tests/issue_199_regressions.rs +0 -20
  163. data/vendor/html-to-markdown-rs/tests/issue_200_regressions.rs +0 -62
  164. data/vendor/html-to-markdown-rs/tests/issue_212_regressions.rs +0 -68
  165. data/vendor/html-to-markdown-rs/tests/issue_216_217_regressions.rs +0 -87
  166. data/vendor/html-to-markdown-rs/tests/issue_336_regressions.rs +0 -74
  167. data/vendor/html-to-markdown-rs/tests/issue_339_regressions.rs +0 -92
  168. data/vendor/html-to-markdown-rs/tests/issue_347_regressions.rs +0 -154
  169. data/vendor/html-to-markdown-rs/tests/issue_348_visitor_plain.rs +0 -93
  170. data/vendor/html-to-markdown-rs/tests/json_ld_script_extraction.rs +0 -44
  171. data/vendor/html-to-markdown-rs/tests/lists_test.rs +0 -199
  172. data/vendor/html-to-markdown-rs/tests/plain_output_test.rs +0 -273
  173. data/vendor/html-to-markdown-rs/tests/preprocessing_tests.rs +0 -61
  174. data/vendor/html-to-markdown-rs/tests/reference_links_test.rs +0 -169
  175. data/vendor/html-to-markdown-rs/tests/sectioning_elements_test.rs +0 -137
  176. data/vendor/html-to-markdown-rs/tests/skip_images_test.rs +0 -522
  177. data/vendor/html-to-markdown-rs/tests/tables_test.rs +0 -743
  178. data/vendor/html-to-markdown-rs/tests/test_custom_elements.rs +0 -41
  179. data/vendor/html-to-markdown-rs/tests/test_issue_187.rs +0 -204
  180. data/vendor/html-to-markdown-rs/tests/test_issue_218.rs +0 -68
  181. data/vendor/html-to-markdown-rs/tests/test_issue_277.rs +0 -77
  182. data/vendor/html-to-markdown-rs/tests/test_max_depth.rs +0 -82
  183. data/vendor/html-to-markdown-rs/tests/test_nested_simple.rs +0 -45
  184. data/vendor/html-to-markdown-rs/tests/test_script_style_stripping.rs +0 -396
  185. data/vendor/html-to-markdown-rs/tests/test_spa_bisect.rs +0 -34
  186. data/vendor/html-to-markdown-rs/tests/visitor_code_integration_test.rs +0 -121
  187. data/vendor/html-to-markdown-rs/tests/visitor_integration_test.rs +0 -1190
  188. data/vendor/html-to-markdown-rs/tests/xml_tables_test.rs +0 -372
@@ -1,398 +0,0 @@
1
- //! Metadata extraction utilities and helpers.
2
-
3
- use super::types::{DocumentMetadata, StructuredData, StructuredDataType, TextDirection};
4
- use std::collections::BTreeMap;
5
-
6
- /// Extract document metadata from collected head metadata.
7
- ///
8
- /// Parses head metadata into structured document metadata,
9
- /// handling special cases like Open Graph, Twitter Card, keywords, etc.
10
- pub(crate) fn extract_document_metadata(
11
- head_metadata: BTreeMap<String, String>,
12
- lang: Option<String>,
13
- dir: Option<String>,
14
- ) -> DocumentMetadata {
15
- let mut doc = DocumentMetadata::default();
16
-
17
- for (raw_key, value) in head_metadata {
18
- let mut key = raw_key.as_str();
19
- let mut replaced_key: Option<String> = None;
20
-
21
- if let Some(stripped) = key.strip_prefix("meta-") {
22
- key = stripped;
23
- }
24
-
25
- if key.as_bytes().contains(&b':') {
26
- replaced_key = Some(key.replace(':', "-"));
27
- key = replaced_key.as_deref().unwrap_or(key);
28
- }
29
-
30
- // Normalize to lowercase for case-insensitive matching (per HTML spec,
31
- // meta name attributes are compared ASCII case-insensitively).
32
- let lower_key = key.to_ascii_lowercase();
33
-
34
- match lower_key.as_str() {
35
- "title" => doc.title = Some(value),
36
- "description" => doc.description = Some(value),
37
- "author" | "creator" | "publisher" => {
38
- if doc.author.is_none() {
39
- doc.author = Some(value);
40
- }
41
- }
42
- "canonical" => doc.canonical_url = Some(value),
43
- "base" | "base-href" => doc.base_href = Some(value),
44
- k if k.starts_with("og-") => {
45
- let og_key = k.trim_start_matches("og-").replace('-', "_");
46
- doc.open_graph.insert(og_key, value);
47
- }
48
- k if k.starts_with("twitter-") => {
49
- let tw_key = k.trim_start_matches("twitter-").replace('-', "_");
50
- doc.twitter_card.insert(tw_key, value);
51
- }
52
- // Dublin Core: DC.* and DCTERMS.* prefixes (dot becomes part of key after meta- strip).
53
- // Map DC/DCTERMS fields to dedicated struct fields where applicable.
54
- k if k.starts_with("dc.") || k.starts_with("dc-") => {
55
- let dc_field = k.trim_start_matches("dc.").trim_start_matches("dc-");
56
- match dc_field {
57
- "title" => {
58
- if doc.title.is_none() {
59
- doc.title = Some(value);
60
- }
61
- }
62
- "description" => {
63
- if doc.description.is_none() {
64
- doc.description = Some(value);
65
- }
66
- }
67
- "creator" | "contributor" | "publisher" => {
68
- if doc.author.is_none() {
69
- doc.author = Some(value);
70
- }
71
- }
72
- "subject" | "keywords" => {
73
- if doc.keywords.is_empty() {
74
- doc.keywords = split_keywords(&value);
75
- }
76
- }
77
- _ => {
78
- let meta_key = format!("dc_{}", dc_field.replace('-', "_"));
79
- doc.meta_tags.insert(meta_key, value);
80
- }
81
- }
82
- }
83
- k if k.starts_with("dcterms.") || k.starts_with("dcterms-") => {
84
- let dc_field = k.trim_start_matches("dcterms.").trim_start_matches("dcterms-");
85
- match dc_field {
86
- "title" | "alternative" => {
87
- if doc.title.is_none() {
88
- doc.title = Some(value);
89
- }
90
- }
91
- "description" | "abstract" => {
92
- if doc.description.is_none() {
93
- doc.description = Some(value);
94
- }
95
- }
96
- "creator" | "contributor" | "publisher" => {
97
- if doc.author.is_none() {
98
- doc.author = Some(value);
99
- }
100
- }
101
- "subject" | "keywords" => {
102
- if doc.keywords.is_empty() {
103
- doc.keywords = split_keywords(&value);
104
- }
105
- }
106
- _ => {
107
- let meta_key = format!("dcterms_{}", dc_field.replace('-', "_"));
108
- doc.meta_tags.insert(meta_key, value);
109
- }
110
- }
111
- }
112
- // All keyword-bearing meta tag variants
113
- "keywords" | "news_keywords" | "citation_keywords" | "subject" | "topic" | "category"
114
- | "classification" => {
115
- if doc.keywords.is_empty() {
116
- doc.keywords = split_keywords(&value);
117
- }
118
- }
119
- _ => {
120
- let meta_key = if key.as_ptr() == raw_key.as_ptr() && key.len() == raw_key.len() {
121
- raw_key
122
- } else if let Some(replaced) = replaced_key {
123
- replaced
124
- } else {
125
- key.to_string()
126
- };
127
- doc.meta_tags.insert(meta_key, value);
128
- }
129
- }
130
- }
131
-
132
- if let Some(lang) = lang {
133
- doc.language = Some(lang);
134
- }
135
-
136
- if let Some(dir) = dir {
137
- if let Some(parsed_dir) = TextDirection::parse(&dir) {
138
- doc.text_direction = Some(parsed_dir);
139
- }
140
- }
141
-
142
- doc
143
- }
144
-
145
- /// Split a comma-separated keywords string into a `Vec<String>`.
146
- fn split_keywords(value: &str) -> Vec<String> {
147
- value
148
- .split(',')
149
- .map(|s| s.trim().to_string())
150
- .filter(|s| !s.is_empty())
151
- .collect()
152
- }
153
-
154
- /// Extract structured data blocks into `StructuredData` items.
155
- pub(crate) fn extract_structured_data(json_ld: Vec<String>) -> Vec<StructuredData> {
156
- let mut result = Vec::with_capacity(json_ld.len());
157
-
158
- for json_str in json_ld {
159
- let schema_type = scan_schema_type(&json_str)
160
- .or_else(|| {
161
- if json_str.contains("\"@type\"") {
162
- serde_json::from_str::<serde_json::Value>(&json_str).ok().and_then(|v| {
163
- v.get("@type")
164
- .and_then(|t| t.as_str().map(std::string::ToString::to_string))
165
- })
166
- } else {
167
- None
168
- }
169
- })
170
- .or_else(|| {
171
- if !json_str.contains("\"@graph\"") {
172
- return None;
173
- }
174
-
175
- let value = serde_json::from_str::<serde_json::Value>(&json_str).ok()?;
176
- let graph = value.get("@graph")?;
177
- let items = graph.as_array()?;
178
- items.iter().find_map(|item| {
179
- item.get("@type")
180
- .and_then(|t| t.as_str().map(std::string::ToString::to_string))
181
- })
182
- });
183
-
184
- result.push(StructuredData {
185
- data_type: StructuredDataType::JsonLd,
186
- raw_json: json_str,
187
- schema_type,
188
- });
189
- }
190
-
191
- result
192
- }
193
-
194
- /// Scan for @type in JSON string without full parsing.
195
- fn scan_schema_type(json_str: &str) -> Option<String> {
196
- let needle = "\"@type\"";
197
- let start = json_str.find(needle)? + needle.len();
198
- let bytes = json_str.as_bytes();
199
- let mut i = start;
200
-
201
- while i < bytes.len() && bytes[i].is_ascii_whitespace() {
202
- i += 1;
203
- }
204
- if i >= bytes.len() || bytes[i] != b':' {
205
- return None;
206
- }
207
- i += 1;
208
- while i < bytes.len() && bytes[i].is_ascii_whitespace() {
209
- i += 1;
210
- }
211
- if i >= bytes.len() {
212
- return None;
213
- }
214
-
215
- if bytes[i] == b'[' {
216
- i += 1;
217
- while i < bytes.len() && bytes[i].is_ascii_whitespace() {
218
- i += 1;
219
- }
220
- if i >= bytes.len() || bytes[i] != b'"' {
221
- return None;
222
- }
223
- } else if bytes[i] != b'"' {
224
- return None;
225
- }
226
-
227
- let start_quote = i;
228
- i += 1;
229
- let mut escaped = false;
230
- while i < bytes.len() {
231
- let byte = bytes[i];
232
- if escaped {
233
- escaped = false;
234
- i += 1;
235
- continue;
236
- }
237
- if byte == b'\\' {
238
- escaped = true;
239
- i += 1;
240
- continue;
241
- }
242
- if byte == b'"' {
243
- let end_quote = i;
244
- let slice = &json_str[start_quote..=end_quote];
245
- return serde_json::from_str::<String>(slice).ok();
246
- }
247
- i += 1;
248
- }
249
-
250
- None
251
- }
252
-
253
- #[cfg(test)]
254
- mod tests {
255
- use super::*;
256
-
257
- #[test]
258
- fn test_extract_document_metadata() {
259
- let mut head_metadata = BTreeMap::new();
260
- head_metadata.insert("title".to_string(), "Test Title".to_string());
261
- head_metadata.insert("description".to_string(), "Test Description".to_string());
262
- head_metadata.insert("keywords".to_string(), "rust, testing".to_string());
263
-
264
- let doc = extract_document_metadata(head_metadata, Some("en".to_string()), Some("ltr".to_string()));
265
-
266
- assert_eq!(doc.title, Some("Test Title".to_string()));
267
- assert_eq!(doc.description, Some("Test Description".to_string()));
268
- assert_eq!(doc.keywords, vec!["rust", "testing"]);
269
- assert_eq!(doc.language, Some("en".to_string()));
270
- }
271
-
272
- #[test]
273
- fn test_extract_document_metadata_with_og() {
274
- let mut head_metadata = BTreeMap::new();
275
- head_metadata.insert("og-title".to_string(), "OG Title".to_string());
276
- head_metadata.insert("og-description".to_string(), "OG Description".to_string());
277
-
278
- let doc = extract_document_metadata(head_metadata, None, None);
279
-
280
- assert_eq!(doc.open_graph.get("title"), Some(&"OG Title".to_string()));
281
- assert_eq!(doc.open_graph.get("description"), Some(&"OG Description".to_string()));
282
- }
283
-
284
- #[test]
285
- fn test_keywords_case_insensitive() {
286
- let mut head_metadata = BTreeMap::new();
287
- head_metadata.insert("meta-Keywords".to_string(), "Rust, HTML, Markdown".to_string());
288
-
289
- let doc = extract_document_metadata(head_metadata, None, None);
290
- assert_eq!(doc.keywords, vec!["Rust", "HTML", "Markdown"]);
291
- }
292
-
293
- #[test]
294
- fn test_keywords_dc_subject() {
295
- let mut head_metadata = BTreeMap::new();
296
- head_metadata.insert("meta-DC.subject".to_string(), "weather, forecast".to_string());
297
-
298
- let doc = extract_document_metadata(head_metadata, None, None);
299
- assert_eq!(doc.keywords, vec!["weather", "forecast"]);
300
- }
301
-
302
- #[test]
303
- fn test_keywords_dc_keywords() {
304
- let mut head_metadata = BTreeMap::new();
305
- head_metadata.insert("meta-DC.keywords".to_string(), "climate, data".to_string());
306
-
307
- let doc = extract_document_metadata(head_metadata, None, None);
308
- assert_eq!(doc.keywords, vec!["climate", "data"]);
309
- }
310
-
311
- #[test]
312
- fn test_keywords_dcterms_subject() {
313
- let mut head_metadata = BTreeMap::new();
314
- head_metadata.insert("meta-DCTERMS.subject".to_string(), "science, research".to_string());
315
-
316
- let doc = extract_document_metadata(head_metadata, None, None);
317
- assert_eq!(doc.keywords, vec!["science", "research"]);
318
- }
319
-
320
- #[test]
321
- fn test_keywords_news_keywords() {
322
- let mut head_metadata = BTreeMap::new();
323
- head_metadata.insert("meta-news_keywords".to_string(), "breaking, world".to_string());
324
-
325
- let doc = extract_document_metadata(head_metadata, None, None);
326
- assert_eq!(doc.keywords, vec!["breaking", "world"]);
327
- }
328
-
329
- #[test]
330
- fn test_keywords_citation_keywords() {
331
- let mut head_metadata = BTreeMap::new();
332
- head_metadata.insert("meta-citation_keywords".to_string(), "biology, genetics".to_string());
333
-
334
- let doc = extract_document_metadata(head_metadata, None, None);
335
- assert_eq!(doc.keywords, vec!["biology", "genetics"]);
336
- }
337
-
338
- #[test]
339
- fn test_dc_title_and_description_fallback() {
340
- let mut head_metadata = BTreeMap::new();
341
- head_metadata.insert("meta-DC.title".to_string(), "DC Title".to_string());
342
- head_metadata.insert("meta-DC.description".to_string(), "DC Description".to_string());
343
- head_metadata.insert("meta-DC.creator".to_string(), "DC Author".to_string());
344
-
345
- let doc = extract_document_metadata(head_metadata, None, None);
346
- assert_eq!(doc.title, Some("DC Title".to_string()));
347
- assert_eq!(doc.description, Some("DC Description".to_string()));
348
- assert_eq!(doc.author, Some("DC Author".to_string()));
349
- }
350
-
351
- #[test]
352
- fn test_dc_does_not_override_standard_fields() {
353
- let mut head_metadata = BTreeMap::new();
354
- // Standard fields come first alphabetically in BTreeMap
355
- head_metadata.insert("description".to_string(), "Standard Description".to_string());
356
- head_metadata.insert("meta-DC.description".to_string(), "DC Description".to_string());
357
- head_metadata.insert("title".to_string(), "Standard Title".to_string());
358
- head_metadata.insert("meta-DC.title".to_string(), "DC Title".to_string());
359
-
360
- let doc = extract_document_metadata(head_metadata, None, None);
361
- assert_eq!(doc.title, Some("Standard Title".to_string()));
362
- assert_eq!(doc.description, Some("Standard Description".to_string()));
363
- }
364
-
365
- #[test]
366
- fn test_case_insensitive_title_description_author() {
367
- let mut head_metadata = BTreeMap::new();
368
- head_metadata.insert("meta-Title".to_string(), "My Title".to_string());
369
- head_metadata.insert("meta-Description".to_string(), "My Desc".to_string());
370
- head_metadata.insert("meta-Author".to_string(), "My Author".to_string());
371
-
372
- let doc = extract_document_metadata(head_metadata, None, None);
373
- assert_eq!(doc.title, Some("My Title".to_string()));
374
- assert_eq!(doc.description, Some("My Desc".to_string()));
375
- assert_eq!(doc.author, Some("My Author".to_string()));
376
- }
377
-
378
- #[test]
379
- fn test_dcterms_remaining_go_to_meta_tags() {
380
- let mut head_metadata = BTreeMap::new();
381
- head_metadata.insert("meta-DCTERMS.license".to_string(), "MIT".to_string());
382
-
383
- let doc = extract_document_metadata(head_metadata, None, None);
384
- assert_eq!(doc.meta_tags.get("dcterms_license"), Some(&"MIT".to_string()));
385
- }
386
-
387
- #[test]
388
- fn test_scan_schema_type() {
389
- let json = r#"{"@type":"Article","title":"Test"}"#;
390
- assert_eq!(scan_schema_type(json), Some("Article".to_string()));
391
-
392
- let json_array = r#"{"@type":["Article","NewsArticle"]}"#;
393
- assert_eq!(scan_schema_type(json_array), Some("Article".to_string()));
394
-
395
- let no_type = r#"{"title":"Test"}"#;
396
- assert_eq!(scan_schema_type(no_type), None);
397
- }
398
- }
@@ -1,288 +0,0 @@
1
- #![allow(clippy::cast_precision_loss, clippy::cast_sign_loss, clippy::unused_self)]
2
- //! Metadata extraction for HTML to Markdown conversion.
3
- //!
4
- //! This module provides comprehensive, type-safe metadata extraction during HTML-to-Markdown
5
- //! conversion, enabling content analysis, SEO optimization, and document indexing workflows.
6
- //! Metadata includes:
7
- //! - **Document metadata**: Title, description, author, language, canonical URL, Open Graph, Twitter Card
8
- //! - **Headers**: Heading elements (h1-h6) with hierarchy, IDs, and positions
9
- //! - **Links**: Hyperlinks with type classification (anchor, internal, external, email, phone)
10
- //! - **Images**: Image elements with source, alt text, dimensions, and type (data URI, external, etc.)
11
- //! - **Structured data**: JSON-LD, Microdata, and RDFa blocks
12
- //!
13
- //! The implementation follows a single-pass collector pattern for zero-overhead extraction
14
- //! when metadata features are disabled.
15
- //!
16
- //! # Architecture
17
- //!
18
- //! Metadata extraction uses the [`MetadataCollector`] pattern:
19
- //! - **Single-pass collection**: Metadata is gathered during the primary tree traversal without additional passes
20
- //! - **Zero overhead when disabled**: Entire module can be compiled out via feature flags
21
- //! - **Configurable granularity**: Use [`MetadataConfig`] to select which metadata types to extract
22
- //! - **Type-safe APIs**: All metadata types are enum-based with exhaustive matching
23
- //! - **Memory-bounded**: Size limits prevent memory exhaustion from adversarial documents
24
- //! - **Pre-allocated buffers**: Typical documents (32 headers, 64 links, 16 images) handled efficiently
25
- //!
26
- //! # Type Overview
27
- //!
28
- //! ## Enumerations
29
- //!
30
- //! - [`TextDirection`]: Document directionality (LTR, RTL, Auto)
31
- //! - [`LinkType`]: Link classification (Anchor, Internal, External, Email, Phone, Other)
32
- //! - [`ImageType`]: Image source type (DataUri, External, Relative, InlineSvg)
33
- //! - [`StructuredDataType`]: Structured data format (JsonLd, Microdata, RDFa)
34
- //!
35
- //! ## Structures
36
- //!
37
- //! - [`DocumentMetadata`]: Head-level metadata with maps for Open Graph and Twitter Card
38
- //! - [`HeaderMetadata`]: Heading element with level (1-6), text, ID, hierarchy depth, and position
39
- //! - [`LinkMetadata`]: Hyperlink with href, text, title, type, rel attributes, and custom attributes
40
- //! - [`ImageMetadata`]: Image element with src, alt, title, dimensions, type, and attributes
41
- //! - [`StructuredData`]: Structured data block with type and raw JSON
42
- //! - [`MetadataConfig`]: Configuration controlling extraction granularity and size limits
43
- //! - [`HtmlMetadata`]: Top-level result containing all extracted metadata
44
- //!
45
- //! # Examples
46
- //!
47
- //! ## Basic Usage with `convert()`
48
- //!
49
- //! ```text
50
- //! use html_to_markdown_rs::convert;
51
- //!
52
- //! let html = r#"
53
- //! <html lang="en">
54
- //! <head>
55
- //! <title>My Article</title>
56
- //! <meta name="description" content="An interesting read">
57
- //! </head>
58
- //! <body>
59
- //! <h1 id="main">Title</h1>
60
- //! <a href="https://example.com">External Link</a>
61
- //! <img src="photo.jpg" alt="A photo">
62
- //! </body>
63
- //! </html>
64
- //! "#;
65
- //!
66
- //! let result = convert(html, None)?;
67
- //! let metadata = result.metadata.unwrap();
68
- //!
69
- //! // Access document metadata
70
- //! assert_eq!(metadata.document.title, Some("My Article".to_string()));
71
- //! assert_eq!(metadata.document.language, Some("en".to_string()));
72
- //!
73
- //! // Access headers
74
- //! assert_eq!(metadata.headers.len(), 1);
75
- //! assert_eq!(metadata.headers[0].level, 1);
76
- //! assert_eq!(metadata.headers[0].id, Some("main".to_string()));
77
- //!
78
- //! // Access links
79
- //! assert_eq!(metadata.links.len(), 1);
80
- //! assert_eq!(metadata.links[0].link_type, LinkType::External);
81
- //!
82
- //! // Access images
83
- //! assert_eq!(metadata.images.len(), 1);
84
- //! assert_eq!(metadata.images[0].image_type, ImageType::Relative);
85
- //! # Ok::<(), html_to_markdown_rs::ConversionError>(())
86
- //! ```
87
- //!
88
- //! ## Selective Extraction
89
- //!
90
- //! ```text
91
- //! use html_to_markdown_rs::{convert, ConversionOptions};
92
- //!
93
- //! let options = ConversionOptions {
94
- //! extract_metadata: false, // Disable metadata extraction
95
- //! ..Default::default()
96
- //! };
97
- //!
98
- //! let result = convert(html, Some(options))?;
99
- //! assert!(result.metadata.is_none()); // Metadata not extracted
100
- //! # Ok::<(), html_to_markdown_rs::ConversionError>(())
101
- //! ```
102
- //!
103
- //! ## Analyzing Link Types
104
- //!
105
- //! ```text
106
- //! use html_to_markdown_rs::convert;
107
- //! use html_to_markdown_rs::metadata::LinkType;
108
- //!
109
- //! let result = convert(html, None)?;
110
- //! let metadata = result.metadata.unwrap();
111
- //!
112
- //! for link in &metadata.links {
113
- //! match link.link_type {
114
- //! LinkType::External => println!("External: {}", link.href),
115
- //! LinkType::Internal => println!("Internal: {}", link.href),
116
- //! LinkType::Anchor => println!("Anchor: {}", link.href),
117
- //! LinkType::Email => println!("Email: {}", link.href),
118
- //! _ => {}
119
- //! }
120
- //! }
121
- //! # Ok::<(), html_to_markdown_rs::ConversionError>(())
122
- //! ```
123
- //!
124
- //! # Serialization
125
- //!
126
- //! All types in this module support serialization via `serde` when the `metadata` feature is enabled.
127
- //! This enables easy export to JSON, YAML, or other formats:
128
- //!
129
- //! ```text
130
- //! use html_to_markdown_rs::convert;
131
- //!
132
- //! let result = convert(html, None)?;
133
- //! if let Some(metadata) = &result.metadata {
134
- //! let json = serde_json::to_string_pretty(metadata)?;
135
- //! println!("{}", json);
136
- //! }
137
- //! # Ok::<(), Box<dyn std::error::Error>>(())
138
- //! ```
139
-
140
- pub mod collector;
141
- pub mod config;
142
- pub mod extraction;
143
- pub mod types;
144
-
145
- // Re-export public types
146
- pub use collector::MetadataCollector;
147
- pub use config::{DEFAULT_MAX_STRUCTURED_DATA_SIZE, MetadataConfig, MetadataConfigUpdate};
148
- pub use types::{
149
- DocumentMetadata, HeaderMetadata, HtmlMetadata, ImageMetadata, ImageType, LinkMetadata, LinkType, StructuredData,
150
- StructuredDataType, TextDirection,
151
- };
152
-
153
- // Internal handle type for shared mutable access during tree traversal
154
- use std::cell::RefCell;
155
- use std::rc::Rc;
156
-
157
- /// Handle to a metadata collector via reference-counted mutable cell.
158
- ///
159
- /// Used internally for sharing collector state across the tree traversal.
160
- ///
161
- /// # Examples
162
- ///
163
- /// ```text
164
- /// let collector = MetadataCollector::new(MetadataConfig::default());
165
- /// let handle = Rc::new(RefCell::new(collector));
166
- ///
167
- /// // In tree walk, can be passed and borrowed
168
- /// handle.borrow_mut().add_header(1, "Title".to_string(), None, 0, 100);
169
- ///
170
- /// let metadata = handle.take().finish();
171
- /// ```
172
- #[allow(dead_code)]
173
- pub(crate) type MetadataCollectorHandle = Rc<RefCell<MetadataCollector>>;
174
-
175
- #[cfg(test)]
176
- mod tests {
177
- use super::*;
178
-
179
- #[test]
180
- fn test_text_direction_parse() {
181
- assert_eq!(TextDirection::parse("ltr"), Some(TextDirection::LeftToRight));
182
- assert_eq!(TextDirection::parse("rtl"), Some(TextDirection::RightToLeft));
183
- assert_eq!(TextDirection::parse("auto"), Some(TextDirection::Auto));
184
- assert_eq!(TextDirection::parse("invalid"), None);
185
- assert_eq!(TextDirection::parse("LTR"), Some(TextDirection::LeftToRight));
186
- }
187
-
188
- #[test]
189
- fn test_text_direction_display() {
190
- assert_eq!(TextDirection::LeftToRight.to_string(), "ltr");
191
- assert_eq!(TextDirection::RightToLeft.to_string(), "rtl");
192
- assert_eq!(TextDirection::Auto.to_string(), "auto");
193
- }
194
-
195
- #[test]
196
- fn test_link_classification() {
197
- assert_eq!(LinkMetadata::classify_link("#section"), LinkType::Anchor);
198
- assert_eq!(LinkMetadata::classify_link("mailto:test@example.com"), LinkType::Email);
199
- assert_eq!(LinkMetadata::classify_link("tel:+1234567890"), LinkType::Phone);
200
- assert_eq!(LinkMetadata::classify_link("https://example.com"), LinkType::External);
201
- assert_eq!(LinkMetadata::classify_link("http://example.com"), LinkType::External);
202
- assert_eq!(LinkMetadata::classify_link("/path/to/page"), LinkType::Internal);
203
- assert_eq!(LinkMetadata::classify_link("../relative"), LinkType::Internal);
204
- assert_eq!(LinkMetadata::classify_link("./same"), LinkType::Internal);
205
- }
206
-
207
- #[test]
208
- fn test_header_validation() {
209
- let valid = HeaderMetadata {
210
- level: 3,
211
- text: "Title".to_string(),
212
- id: None,
213
- depth: 2,
214
- html_offset: 100,
215
- };
216
- assert!(valid.is_valid());
217
-
218
- let invalid_high = HeaderMetadata {
219
- level: 7,
220
- text: "Title".to_string(),
221
- id: None,
222
- depth: 2,
223
- html_offset: 100,
224
- };
225
- assert!(!invalid_high.is_valid());
226
-
227
- let invalid_low = HeaderMetadata {
228
- level: 0,
229
- text: "Title".to_string(),
230
- id: None,
231
- depth: 2,
232
- html_offset: 100,
233
- };
234
- assert!(!invalid_low.is_valid());
235
- }
236
-
237
- #[test]
238
- fn test_document_metadata_default() {
239
- let doc = DocumentMetadata::default();
240
-
241
- assert!(doc.title.is_none());
242
- assert!(doc.description.is_none());
243
- assert!(doc.keywords.is_empty());
244
- assert!(doc.open_graph.is_empty());
245
- assert!(doc.twitter_card.is_empty());
246
- assert!(doc.meta_tags.is_empty());
247
- }
248
-
249
- #[test]
250
- fn test_image_type_classification() {
251
- let data_uri = ImageMetadata {
252
- src: "data:image/png;base64,iVBORw0KG...".to_string(),
253
- alt: None,
254
- title: None,
255
- dimensions: None,
256
- image_type: ImageType::DataUri,
257
- attributes: Default::default(),
258
- };
259
- assert_eq!(data_uri.image_type, ImageType::DataUri);
260
-
261
- let external = ImageMetadata {
262
- src: "https://example.com/image.jpg".to_string(),
263
- alt: None,
264
- title: None,
265
- dimensions: None,
266
- image_type: ImageType::External,
267
- attributes: Default::default(),
268
- };
269
- assert_eq!(external.image_type, ImageType::External);
270
- }
271
-
272
- #[test]
273
- fn test_link_type_display() {
274
- assert_eq!(LinkType::Anchor.to_string(), "anchor");
275
- assert_eq!(LinkType::Internal.to_string(), "internal");
276
- assert_eq!(LinkType::External.to_string(), "external");
277
- assert_eq!(LinkType::Email.to_string(), "email");
278
- assert_eq!(LinkType::Phone.to_string(), "phone");
279
- assert_eq!(LinkType::Other.to_string(), "other");
280
- }
281
-
282
- #[test]
283
- fn test_structured_data_type_display() {
284
- assert_eq!(StructuredDataType::JsonLd.to_string(), "json_ld");
285
- assert_eq!(StructuredDataType::Microdata.to_string(), "microdata");
286
- assert_eq!(StructuredDataType::RDFa.to_string(), "rdfa");
287
- }
288
- }