html-to-markdown 2.30.0-x86_64-linux → 3.0.0-x86_64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +4 -14
- data/README.md +37 -50
- data/lib/html_to_markdown/version.rb +1 -1
- data/lib/html_to_markdown.rb +13 -194
- data/lib/html_to_markdown_rb.so +0 -0
- data/sig/html_to_markdown.rbs +12 -373
- metadata +2 -8
- data/bin/benchmark.rb +0 -232
- data/spec/convert_spec.rb +0 -77
- data/spec/convert_with_tables_spec.rb +0 -194
- data/spec/metadata_extraction_spec.rb +0 -437
- data/spec/visitor_issue_187_spec.rb +0 -605
- data/spec/visitor_spec.rb +0 -1149
data/sig/html_to_markdown.rbs
CHANGED
|
@@ -2,36 +2,6 @@
|
|
|
2
2
|
module HtmlToMarkdown
|
|
3
3
|
VERSION: String
|
|
4
4
|
|
|
5
|
-
# Opaque handle for reusable conversion options
|
|
6
|
-
class Options
|
|
7
|
-
end
|
|
8
|
-
|
|
9
|
-
# Visitor context information passed to visitor callbacks
|
|
10
|
-
class NodeContext
|
|
11
|
-
attr_reader node_type: Symbol
|
|
12
|
-
attr_reader tag_name: String
|
|
13
|
-
attr_reader attributes: Hash[String, String]
|
|
14
|
-
attr_reader depth: Integer
|
|
15
|
-
attr_reader index_in_parent: Integer
|
|
16
|
-
attr_reader parent_tag: String | nil
|
|
17
|
-
attr_reader is_inline: bool
|
|
18
|
-
end
|
|
19
|
-
|
|
20
|
-
# Result of a visitor callback
|
|
21
|
-
type visitor_result = {
|
|
22
|
-
type: :continue,
|
|
23
|
-
} | {
|
|
24
|
-
type: :custom,
|
|
25
|
-
output: String,
|
|
26
|
-
} | {
|
|
27
|
-
type: :skip,
|
|
28
|
-
} | {
|
|
29
|
-
type: :preserve_html,
|
|
30
|
-
} | {
|
|
31
|
-
type: :error,
|
|
32
|
-
message: String,
|
|
33
|
-
}
|
|
34
|
-
|
|
35
5
|
type heading_style = :underlined | :atx | :atx_closed
|
|
36
6
|
type list_indent_type = :spaces | :tabs
|
|
37
7
|
type highlight_style = :double_equal | :html | :bold | :none
|
|
@@ -62,7 +32,6 @@ module HtmlToMarkdown
|
|
|
62
32
|
autolinks?: bool,
|
|
63
33
|
default_title?: bool,
|
|
64
34
|
br_in_tables?: bool,
|
|
65
|
-
hocr_spatial_tables?: bool, # Deprecated since 2.30.0: hOCR support will be removed in v3.
|
|
66
35
|
highlight_style?: highlight_style,
|
|
67
36
|
extract_metadata?: bool,
|
|
68
37
|
whitespace_mode?: whitespace_mode,
|
|
@@ -81,50 +50,14 @@ module HtmlToMarkdown
|
|
|
81
50
|
strip_tags?: Array[String],
|
|
82
51
|
preserve_tags?: Array[String],
|
|
83
52
|
output_format?: output_format,
|
|
84
|
-
skip_images?: bool
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
max_decoded_size_bytes?: Integer,
|
|
89
|
-
filename_prefix?: String?,
|
|
53
|
+
skip_images?: bool,
|
|
54
|
+
include_document_structure?: bool,
|
|
55
|
+
extract_images?: bool,
|
|
56
|
+
max_image_size?: Integer,
|
|
90
57
|
capture_svg?: bool,
|
|
91
58
|
infer_dimensions?: bool
|
|
92
59
|
}
|
|
93
60
|
|
|
94
|
-
type inline_image_format = "png" | "jpeg" | "gif" | "bmp" | "webp" | "svg" | String
|
|
95
|
-
|
|
96
|
-
type inline_image_source = "img_data_uri" | "svg_element"
|
|
97
|
-
|
|
98
|
-
type inline_image = {
|
|
99
|
-
data: String,
|
|
100
|
-
format: inline_image_format,
|
|
101
|
-
filename: String?,
|
|
102
|
-
description: String?,
|
|
103
|
-
dimensions: [Integer, Integer]?,
|
|
104
|
-
source: inline_image_source,
|
|
105
|
-
attributes: Hash[String, String]
|
|
106
|
-
}
|
|
107
|
-
|
|
108
|
-
type inline_image_warning = {
|
|
109
|
-
index: Integer,
|
|
110
|
-
message: String
|
|
111
|
-
}
|
|
112
|
-
|
|
113
|
-
type html_extraction = {
|
|
114
|
-
markdown: String,
|
|
115
|
-
inline_images: Array[inline_image],
|
|
116
|
-
warnings: Array[inline_image_warning]
|
|
117
|
-
}
|
|
118
|
-
|
|
119
|
-
type metadata_config = {
|
|
120
|
-
extract_document?: bool,
|
|
121
|
-
extract_headers?: bool,
|
|
122
|
-
extract_links?: bool,
|
|
123
|
-
extract_images?: bool,
|
|
124
|
-
extract_structured_data?: bool,
|
|
125
|
-
max_structured_data_size?: Integer
|
|
126
|
-
}
|
|
127
|
-
|
|
128
61
|
type text_direction = "ltr" | "rtl" | "auto" | nil
|
|
129
62
|
|
|
130
63
|
type document_metadata = {
|
|
@@ -185,314 +118,20 @@ module HtmlToMarkdown
|
|
|
185
118
|
structured_data: Array[structured_data]
|
|
186
119
|
}
|
|
187
120
|
|
|
188
|
-
|
|
189
|
-
cells: Array[Array[String]],
|
|
190
|
-
markdown: String,
|
|
191
|
-
is_header_row: Array[bool]
|
|
192
|
-
}
|
|
193
|
-
|
|
194
|
-
type table_extraction_result = {
|
|
195
|
-
content: String,
|
|
196
|
-
metadata: extended_metadata?,
|
|
197
|
-
tables: Array[table_data]
|
|
198
|
-
}
|
|
199
|
-
|
|
200
|
-
# Native methods (implemented in Rust via Magnus/rb-sys)
|
|
201
|
-
# These are aliased from the Rust extension and available as both module and instance methods
|
|
121
|
+
# Native method (implemented in Rust via Magnus/rb-sys)
|
|
202
122
|
private
|
|
203
123
|
|
|
204
|
-
def self.native_convert: (String html, conversion_options? options) -> String
|
|
205
|
-
def
|
|
206
|
-
def self.native_convert_with_options: (String html, Options options_handle) -> String
|
|
207
|
-
def self.native_convert_with_inline_images_handle: (
|
|
208
|
-
String html,
|
|
209
|
-
Options options_handle,
|
|
210
|
-
inline_image_config? image_config
|
|
211
|
-
) -> html_extraction
|
|
212
|
-
def self.native_convert_with_inline_images: (
|
|
213
|
-
String html,
|
|
214
|
-
conversion_options? options,
|
|
215
|
-
inline_image_config? image_config
|
|
216
|
-
) -> html_extraction
|
|
217
|
-
def self.native_convert_with_metadata_handle: (
|
|
218
|
-
String html,
|
|
219
|
-
Options options_handle,
|
|
220
|
-
metadata_config? metadata_config
|
|
221
|
-
) -> [String, extended_metadata]
|
|
222
|
-
def self.native_convert_with_metadata: (
|
|
223
|
-
String html,
|
|
224
|
-
conversion_options? options,
|
|
225
|
-
metadata_config? metadata_config
|
|
226
|
-
) -> [String, extended_metadata]
|
|
227
|
-
def self.native_convert_with_visitor: (
|
|
228
|
-
String html,
|
|
229
|
-
conversion_options? options,
|
|
230
|
-
visitor? visitor
|
|
231
|
-
) -> String
|
|
232
|
-
def self.native_convert_with_tables: (
|
|
233
|
-
String html,
|
|
234
|
-
conversion_options? options,
|
|
235
|
-
metadata_config? metadata_config
|
|
236
|
-
) -> table_extraction_result
|
|
237
|
-
|
|
238
|
-
def native_convert: (String html, conversion_options? options) -> String
|
|
239
|
-
def native_options: (conversion_options? options_hash) -> Options
|
|
240
|
-
def native_convert_with_options: (String html, Options options_handle) -> String
|
|
241
|
-
def native_convert_with_inline_images_handle: (
|
|
242
|
-
String html,
|
|
243
|
-
Options options_handle,
|
|
244
|
-
inline_image_config? image_config
|
|
245
|
-
) -> html_extraction
|
|
246
|
-
def native_convert_with_inline_images: (
|
|
247
|
-
String html,
|
|
248
|
-
conversion_options? options,
|
|
249
|
-
inline_image_config? image_config
|
|
250
|
-
) -> html_extraction
|
|
251
|
-
def native_convert_with_metadata_handle: (
|
|
252
|
-
String html,
|
|
253
|
-
Options options_handle,
|
|
254
|
-
metadata_config? metadata_config
|
|
255
|
-
) -> [String, extended_metadata]
|
|
256
|
-
def native_convert_with_metadata: (
|
|
257
|
-
String html,
|
|
258
|
-
conversion_options? options,
|
|
259
|
-
metadata_config? metadata_config
|
|
260
|
-
) -> [String, extended_metadata]
|
|
261
|
-
def native_convert_with_visitor: (
|
|
262
|
-
String html,
|
|
263
|
-
conversion_options? options,
|
|
264
|
-
visitor? visitor
|
|
265
|
-
) -> String
|
|
266
|
-
def native_convert_with_tables: (
|
|
267
|
-
String html,
|
|
268
|
-
conversion_options? options,
|
|
269
|
-
metadata_config? metadata_config
|
|
270
|
-
) -> table_extraction_result
|
|
271
|
-
|
|
272
|
-
# Visitor interface for customizing conversion behavior
|
|
273
|
-
type visitor = Object
|
|
124
|
+
def self.native_convert: (String html, conversion_options? options) -> Hash[String, untyped]
|
|
125
|
+
def native_convert: (String html, conversion_options? options) -> Hash[String, untyped]
|
|
274
126
|
|
|
275
127
|
public
|
|
276
128
|
|
|
277
|
-
# Convert HTML to Markdown with
|
|
278
|
-
#
|
|
279
|
-
# The optional visitor parameter allows customization of conversion behavior for specific elements.
|
|
280
|
-
# When both options and visitor are provided, the visitor can override default conversions.
|
|
281
|
-
#
|
|
282
|
-
# Args:
|
|
283
|
-
# html: HTML string to convert
|
|
284
|
-
# options: Optional conversion configuration
|
|
285
|
-
# visitor: Optional visitor object for customizing conversion
|
|
286
|
-
#
|
|
287
|
-
# Returns:
|
|
288
|
-
# markdown: String - Converted markdown output
|
|
289
|
-
#
|
|
290
|
-
# Example:
|
|
291
|
-
# markdown = HtmlToMarkdown.convert(html, { wrap: true }, my_visitor)
|
|
292
|
-
def self.convert: (String html, ?conversion_options options, ?visitor visitor) -> String
|
|
293
|
-
|
|
294
|
-
# Create a reusable options handle for performance
|
|
295
|
-
def self.options: (?conversion_options options_hash) -> Options
|
|
296
|
-
|
|
297
|
-
# Convert HTML using a pre-built options handle
|
|
298
|
-
def self.convert_with_options: (String html, Options options_handle) -> String
|
|
299
|
-
def self.convert_with_inline_images_handle: (
|
|
300
|
-
String html,
|
|
301
|
-
Options options_handle,
|
|
302
|
-
?inline_image_config image_config
|
|
303
|
-
) -> html_extraction
|
|
304
|
-
|
|
305
|
-
# Convert HTML with inline image extraction
|
|
306
|
-
#
|
|
307
|
-
# Optionally accepts a visitor for customizing conversion behavior.
|
|
308
|
-
#
|
|
309
|
-
# Args:
|
|
310
|
-
# html: HTML string to convert
|
|
311
|
-
# options: Optional conversion configuration
|
|
312
|
-
# image_config: Optional inline image extraction configuration
|
|
313
|
-
# visitor: Optional visitor object for customizing conversion
|
|
314
|
-
#
|
|
315
|
-
# Returns:
|
|
316
|
-
# html_extraction: Hash containing markdown, inline_images array, and warnings array
|
|
317
|
-
#
|
|
318
|
-
# Example:
|
|
319
|
-
# result = HtmlToMarkdown.convert_with_inline_images(html, { wrap: true }, image_config, my_visitor)
|
|
320
|
-
def self.convert_with_inline_images: (
|
|
321
|
-
String html,
|
|
322
|
-
?conversion_options options,
|
|
323
|
-
?inline_image_config image_config,
|
|
324
|
-
?visitor visitor
|
|
325
|
-
) -> html_extraction
|
|
326
|
-
|
|
327
|
-
# Convert HTML to Markdown with a custom visitor (deprecated)
|
|
328
|
-
#
|
|
329
|
-
# DEPRECATED: Use convert() with the optional visitor parameter instead.
|
|
330
|
-
# This method is maintained for backward compatibility.
|
|
331
|
-
#
|
|
332
|
-
# All convert functions now accept optional visitors:
|
|
333
|
-
# - convert(html, options, visitor)
|
|
334
|
-
# - convert_with_inline_images(html, options, image_config, visitor)
|
|
335
|
-
# - convert_with_metadata(html, options, metadata_config, visitor)
|
|
336
|
-
#
|
|
337
|
-
# The visitor object can implement any of the following methods:
|
|
338
|
-
# - visit_element_start(ctx) -> visitor_result
|
|
339
|
-
# - visit_element_end(ctx, output) -> visitor_result
|
|
340
|
-
# - visit_text(ctx, text) -> visitor_result
|
|
341
|
-
# - visit_link(ctx, href, text, title) -> visitor_result
|
|
342
|
-
# - visit_image(ctx, src, alt, title) -> visitor_result
|
|
343
|
-
# - visit_heading(ctx, level, text, id) -> visitor_result
|
|
344
|
-
# - visit_code_block(ctx, lang, code) -> visitor_result
|
|
345
|
-
# - visit_code_inline(ctx, code) -> visitor_result
|
|
346
|
-
# - visit_list_item(ctx, ordered, marker, text) -> visitor_result
|
|
347
|
-
# - visit_list_start(ctx, ordered) -> visitor_result
|
|
348
|
-
# - visit_list_end(ctx, ordered, output) -> visitor_result
|
|
349
|
-
# - visit_table_start(ctx) -> visitor_result
|
|
350
|
-
# - visit_table_row(ctx, cells, is_header) -> visitor_result
|
|
351
|
-
# - visit_table_end(ctx, output) -> visitor_result
|
|
352
|
-
# - visit_blockquote(ctx, content, depth) -> visitor_result
|
|
353
|
-
# - visit_strong(ctx, text) -> visitor_result
|
|
354
|
-
# - visit_emphasis(ctx, text) -> visitor_result
|
|
355
|
-
# - visit_strikethrough(ctx, text) -> visitor_result
|
|
356
|
-
# - visit_underline(ctx, text) -> visitor_result
|
|
357
|
-
# - visit_subscript(ctx, text) -> visitor_result
|
|
358
|
-
# - visit_superscript(ctx, text) -> visitor_result
|
|
359
|
-
# - visit_mark(ctx, text) -> visitor_result
|
|
360
|
-
# - visit_line_break(ctx) -> visitor_result
|
|
361
|
-
# - visit_horizontal_rule(ctx) -> visitor_result
|
|
362
|
-
# - visit_custom_element(ctx, tag_name, html) -> visitor_result
|
|
363
|
-
# - visit_definition_list_start(ctx) -> visitor_result
|
|
364
|
-
# - visit_definition_term(ctx, text) -> visitor_result
|
|
365
|
-
# - visit_definition_description(ctx, text) -> visitor_result
|
|
366
|
-
# - visit_definition_list_end(ctx, output) -> visitor_result
|
|
367
|
-
# - visit_form(ctx, action, method) -> visitor_result
|
|
368
|
-
# - visit_input(ctx, input_type, name, value) -> visitor_result
|
|
369
|
-
# - visit_button(ctx, text) -> visitor_result
|
|
370
|
-
# - visit_audio(ctx, src) -> visitor_result
|
|
371
|
-
# - visit_video(ctx, src) -> visitor_result
|
|
372
|
-
# - visit_iframe(ctx, src) -> visitor_result
|
|
373
|
-
# - visit_details(ctx, open) -> visitor_result
|
|
374
|
-
# - visit_summary(ctx, text) -> visitor_result
|
|
375
|
-
# - visit_figure_start(ctx) -> visitor_result
|
|
376
|
-
# - visit_figcaption(ctx, text) -> visitor_result
|
|
377
|
-
# - visit_figure_end(ctx, output) -> visitor_result
|
|
378
|
-
#
|
|
379
|
-
# Each method should return a Hash with at least :type key:
|
|
380
|
-
# { type: :continue } - Continue with default behavior
|
|
381
|
-
# { type: :custom, output: "..." } - Replace with custom markdown
|
|
382
|
-
# { type: :skip } - Skip this element entirely
|
|
383
|
-
# { type: :preserve_html } - Keep original HTML
|
|
384
|
-
# { type: :error, message: "..." } - Stop conversion with error
|
|
385
|
-
#
|
|
386
|
-
# Args:
|
|
387
|
-
# html: HTML string to convert
|
|
388
|
-
# options: Optional conversion configuration
|
|
389
|
-
# visitor: Visitor object that responds to visitor callback methods
|
|
390
|
-
#
|
|
391
|
-
# Returns:
|
|
392
|
-
# markdown: String - Converted markdown output
|
|
393
|
-
#
|
|
394
|
-
# Example:
|
|
395
|
-
# class MyVisitor
|
|
396
|
-
# def visit_link(ctx, href, text, title = nil)
|
|
397
|
-
# { type: :custom, output: "[#{text}](#{href})" }
|
|
398
|
-
# end
|
|
399
|
-
# end
|
|
400
|
-
#
|
|
401
|
-
# HtmlToMarkdown.convert_with_visitor(html, visitor: MyVisitor.new)
|
|
402
|
-
def self.convert_with_visitor: (String html, ?conversion_options options, visitor: visitor) -> String
|
|
403
|
-
|
|
404
|
-
# Convert HTML to Markdown with metadata extraction
|
|
405
|
-
#
|
|
406
|
-
# Extracts comprehensive metadata (headers, links, images, structured data) during conversion.
|
|
407
|
-
# Optionally accepts a visitor for customizing conversion behavior.
|
|
408
|
-
#
|
|
409
|
-
# Args:
|
|
410
|
-
# html: HTML string to convert
|
|
411
|
-
# options: Optional conversion configuration
|
|
412
|
-
# metadata_config: Optional metadata extraction configuration
|
|
413
|
-
# visitor: Optional visitor object for customizing conversion
|
|
414
|
-
#
|
|
415
|
-
# Returns:
|
|
416
|
-
# Array containing:
|
|
417
|
-
# - [0] markdown: String - Converted markdown output
|
|
418
|
-
# - [1] metadata: Hash - Extracted metadata with document, headers, links, images, structured_data
|
|
419
|
-
#
|
|
420
|
-
# The metadata hash contains:
|
|
421
|
-
# - document: Document-level metadata (title, description, lang, etc.)
|
|
422
|
-
# - headers: List of header elements with hierarchy
|
|
423
|
-
# - links: List of extracted hyperlinks with classification
|
|
424
|
-
# - images: List of extracted images with metadata
|
|
425
|
-
# - structured_data: List of JSON-LD, Microdata, or RDFa blocks
|
|
426
|
-
#
|
|
427
|
-
# Example:
|
|
428
|
-
# html = '<html lang="en"><head><title>Test</title></head><body><h1>Hello</h1></body></html>'
|
|
429
|
-
# markdown, metadata = HtmlToMarkdown.convert_with_metadata(html)
|
|
430
|
-
# puts "Title: #{metadata['document']['title']}"
|
|
431
|
-
# puts "Headers: #{metadata['headers'].length}"
|
|
432
|
-
#
|
|
433
|
-
# Example with visitor:
|
|
434
|
-
# markdown, metadata = HtmlToMarkdown.convert_with_metadata(html, options, metadata_config, my_visitor)
|
|
435
|
-
def self.convert_with_metadata: (
|
|
436
|
-
String html,
|
|
437
|
-
?conversion_options options,
|
|
438
|
-
?metadata_config metadata_config,
|
|
439
|
-
?visitor visitor
|
|
440
|
-
) -> [String, extended_metadata]
|
|
441
|
-
def self.convert_with_metadata_handle: (
|
|
442
|
-
String html,
|
|
443
|
-
Options options_handle,
|
|
444
|
-
?metadata_config metadata_config
|
|
445
|
-
) -> [String, extended_metadata]
|
|
446
|
-
|
|
447
|
-
# Convert HTML and extract tables as structured data
|
|
448
|
-
#
|
|
449
|
-
# Args:
|
|
450
|
-
# html: HTML string to convert
|
|
451
|
-
# options: Optional conversion configuration
|
|
452
|
-
# metadata_config: Optional metadata extraction configuration
|
|
453
|
-
#
|
|
454
|
-
# Returns:
|
|
455
|
-
# table_extraction_result: Hash containing content, metadata, and tables array
|
|
129
|
+
# Convert HTML to Markdown, returning a Hash with content, metadata, tables, images, and warnings.
|
|
456
130
|
#
|
|
457
131
|
# Example:
|
|
458
|
-
# result = HtmlToMarkdown.
|
|
459
|
-
|
|
460
|
-
def self.convert_with_tables: (
|
|
461
|
-
String html,
|
|
462
|
-
?conversion_options options,
|
|
463
|
-
?metadata_config metadata_config
|
|
464
|
-
) -> table_extraction_result
|
|
132
|
+
# result = HtmlToMarkdown.convert(html)
|
|
133
|
+
def self.convert: (String html, ?conversion_options options) -> Hash[String, untyped]
|
|
465
134
|
|
|
466
|
-
# Instance method
|
|
467
|
-
def convert: (String html, ?conversion_options options
|
|
468
|
-
def options: (?conversion_options options_hash) -> Options
|
|
469
|
-
def convert_with_options: (String html, Options options_handle) -> String
|
|
470
|
-
def convert_with_inline_images_handle: (
|
|
471
|
-
String html,
|
|
472
|
-
Options options_handle,
|
|
473
|
-
?inline_image_config image_config
|
|
474
|
-
) -> html_extraction
|
|
475
|
-
def convert_with_inline_images: (
|
|
476
|
-
String html,
|
|
477
|
-
?conversion_options options,
|
|
478
|
-
?inline_image_config image_config,
|
|
479
|
-
?visitor visitor
|
|
480
|
-
) -> html_extraction
|
|
481
|
-
def convert_with_visitor: (String html, ?conversion_options options, visitor: visitor) -> String
|
|
482
|
-
def convert_with_metadata: (
|
|
483
|
-
String html,
|
|
484
|
-
?conversion_options options,
|
|
485
|
-
?metadata_config metadata_config,
|
|
486
|
-
?visitor visitor
|
|
487
|
-
) -> [String, extended_metadata]
|
|
488
|
-
def convert_with_metadata_handle: (
|
|
489
|
-
String html,
|
|
490
|
-
Options options_handle,
|
|
491
|
-
?metadata_config metadata_config
|
|
492
|
-
) -> [String, extended_metadata]
|
|
493
|
-
def convert_with_tables: (
|
|
494
|
-
String html,
|
|
495
|
-
?conversion_options options,
|
|
496
|
-
?metadata_config metadata_config
|
|
497
|
-
) -> table_extraction_result
|
|
135
|
+
# Instance method version (created by module_function)
|
|
136
|
+
def convert: (String html, ?conversion_options options) -> Hash[String, untyped]
|
|
498
137
|
end
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: html-to-markdown
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version:
|
|
4
|
+
version: 3.0.0
|
|
5
5
|
platform: x86_64-linux
|
|
6
6
|
authors:
|
|
7
7
|
- Na'aman Hirschfeld
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-03-
|
|
11
|
+
date: 2026-03-30 00:00:00.000000000 Z
|
|
12
12
|
dependencies: []
|
|
13
13
|
description: |-
|
|
14
14
|
html-to-markdown is a native Ruby extension built on the shared Rust engine that powers the html-to-markdown project.
|
|
@@ -29,7 +29,6 @@ files:
|
|
|
29
29
|
- README.md
|
|
30
30
|
- Rakefile
|
|
31
31
|
- Steepfile
|
|
32
|
-
- bin/benchmark.rb
|
|
33
32
|
- exe/html-to-markdown
|
|
34
33
|
- html-to-markdown-rb.gemspec
|
|
35
34
|
- lib/html_to_markdown.rb
|
|
@@ -42,12 +41,7 @@ files:
|
|
|
42
41
|
- sig/html_to_markdown/cli_proxy.rbs
|
|
43
42
|
- sig/open3.rbs
|
|
44
43
|
- spec/cli_proxy_spec.rb
|
|
45
|
-
- spec/convert_spec.rb
|
|
46
|
-
- spec/convert_with_tables_spec.rb
|
|
47
|
-
- spec/metadata_extraction_spec.rb
|
|
48
44
|
- spec/spec_helper.rb
|
|
49
|
-
- spec/visitor_issue_187_spec.rb
|
|
50
|
-
- spec/visitor_spec.rb
|
|
51
45
|
homepage: https://github.com/kreuzberg-dev/html-to-markdown
|
|
52
46
|
licenses:
|
|
53
47
|
- MIT
|
data/bin/benchmark.rb
DELETED
|
@@ -1,232 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env ruby
|
|
2
|
-
# frozen_string_literal: true
|
|
3
|
-
|
|
4
|
-
require 'optparse'
|
|
5
|
-
require 'time'
|
|
6
|
-
|
|
7
|
-
$LOAD_PATH.unshift(File.expand_path('../lib', __dir__))
|
|
8
|
-
require 'html_to_markdown'
|
|
9
|
-
|
|
10
|
-
def json_escape(value)
|
|
11
|
-
value.to_s.gsub(/["\\\n\r]/) do |char|
|
|
12
|
-
case char
|
|
13
|
-
when '"', '\\'
|
|
14
|
-
"\\#{char}"
|
|
15
|
-
when "\n"
|
|
16
|
-
'\\n'
|
|
17
|
-
when "\r"
|
|
18
|
-
'\\r'
|
|
19
|
-
end
|
|
20
|
-
end
|
|
21
|
-
end
|
|
22
|
-
|
|
23
|
-
options = {
|
|
24
|
-
iterations: 50,
|
|
25
|
-
format: 'html',
|
|
26
|
-
scenario: 'convert-default',
|
|
27
|
-
visitor: nil
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
OptionParser.new do |parser|
|
|
31
|
-
parser.banner = 'ruby benchmark.rb --file path/to/fixture.html [--iterations 200]'
|
|
32
|
-
|
|
33
|
-
parser.on('--file FILE', 'HTML fixture to convert repeatedly') do |file|
|
|
34
|
-
options[:file] = file
|
|
35
|
-
end
|
|
36
|
-
|
|
37
|
-
parser.on('--iterations N', Integer, 'Number of conversion iterations (default: 50)') do |n|
|
|
38
|
-
options[:iterations] = n.positive? ? n : 1
|
|
39
|
-
end
|
|
40
|
-
|
|
41
|
-
parser.on('--scenario SCENARIO', 'Scenario to benchmark') do |scenario|
|
|
42
|
-
options[:scenario] = scenario
|
|
43
|
-
end
|
|
44
|
-
|
|
45
|
-
parser.on('--format FORMAT', 'Fixture format (html or hocr)') do |format|
|
|
46
|
-
options[:format] = format.downcase
|
|
47
|
-
end
|
|
48
|
-
|
|
49
|
-
parser.on('--visitor VISITOR', 'Visitor type (noop, simple, custom, complex)') do |visitor|
|
|
50
|
-
options[:visitor] = visitor if %w[noop simple custom complex].include?(visitor)
|
|
51
|
-
end
|
|
52
|
-
end.parse!
|
|
53
|
-
|
|
54
|
-
fixture = options.fetch(:file) do
|
|
55
|
-
warn 'Missing --file parameter'
|
|
56
|
-
exit 1
|
|
57
|
-
end
|
|
58
|
-
|
|
59
|
-
unless File.exist?(fixture)
|
|
60
|
-
warn "Fixture not found: #{fixture}"
|
|
61
|
-
exit 1
|
|
62
|
-
end
|
|
63
|
-
|
|
64
|
-
unless %w[html hocr].include?(options[:format])
|
|
65
|
-
warn "Unsupported format: #{options[:format]}"
|
|
66
|
-
exit 1
|
|
67
|
-
end
|
|
68
|
-
|
|
69
|
-
supported_scenarios = %w[
|
|
70
|
-
convert-default
|
|
71
|
-
convert-options
|
|
72
|
-
inline-images-default
|
|
73
|
-
inline-images-options
|
|
74
|
-
metadata-default
|
|
75
|
-
metadata-options
|
|
76
|
-
]
|
|
77
|
-
unless supported_scenarios.include?(options[:scenario])
|
|
78
|
-
warn "Unsupported scenario: #{options[:scenario]}"
|
|
79
|
-
exit 1
|
|
80
|
-
end
|
|
81
|
-
|
|
82
|
-
# Visitor factory functions
|
|
83
|
-
def create_noop_visitor
|
|
84
|
-
{
|
|
85
|
-
visit_text: proc { |_ctx, _text| 'continue' },
|
|
86
|
-
visit_heading: proc { |_ctx, _level, _text, _id| 'continue' },
|
|
87
|
-
visit_paragraph: proc { |_ctx, _text| 'continue' },
|
|
88
|
-
visit_link: proc { |_ctx, _href, _text, _title| 'continue' },
|
|
89
|
-
visit_image: proc { |_ctx, _src, _alt, _title| 'continue' },
|
|
90
|
-
visit_strong: proc { |_ctx, _text| 'continue' },
|
|
91
|
-
visit_em: proc { |_ctx, _text| 'continue' },
|
|
92
|
-
visit_code: proc { |_ctx, _text| 'continue' },
|
|
93
|
-
visit_br: proc { |_ctx| 'continue' }
|
|
94
|
-
}
|
|
95
|
-
end
|
|
96
|
-
|
|
97
|
-
def create_simple_visitor
|
|
98
|
-
{
|
|
99
|
-
text_count: 0,
|
|
100
|
-
link_count: 0,
|
|
101
|
-
image_count: 0,
|
|
102
|
-
visit_text: proc { |_ctx, _text| 'continue' },
|
|
103
|
-
visit_heading: proc { |_ctx, _level, _text, _id| 'continue' },
|
|
104
|
-
visit_paragraph: proc { |_ctx, _text| 'continue' },
|
|
105
|
-
visit_link: proc { |_ctx, _href, _text, _title| 'continue' },
|
|
106
|
-
visit_image: proc { |_ctx, _src, _alt, _title| 'continue' },
|
|
107
|
-
visit_strong: proc { |_ctx, _text| 'continue' },
|
|
108
|
-
visit_em: proc { |_ctx, _text| 'continue' },
|
|
109
|
-
visit_code: proc { |_ctx, _text| 'continue' },
|
|
110
|
-
visit_br: proc { |_ctx| 'continue' }
|
|
111
|
-
}
|
|
112
|
-
end
|
|
113
|
-
|
|
114
|
-
def create_custom_visitor
|
|
115
|
-
{
|
|
116
|
-
visit_text: proc { |_ctx, _text| 'continue' },
|
|
117
|
-
visit_heading: proc { |_ctx, _level, _text, _id| 'continue' },
|
|
118
|
-
visit_paragraph: proc { |_ctx, _text| 'continue' },
|
|
119
|
-
visit_link: proc { |_ctx, href, text, _title| ['custom', "LINK[#{text}](#{href})"] },
|
|
120
|
-
visit_image: proc { |_ctx, src, alt, _title| ['custom', ""] },
|
|
121
|
-
visit_strong: proc { |_ctx, _text| 'continue' },
|
|
122
|
-
visit_em: proc { |_ctx, _text| 'continue' },
|
|
123
|
-
visit_code: proc { |_ctx, _text| 'continue' },
|
|
124
|
-
visit_br: proc { |_ctx| 'continue' }
|
|
125
|
-
}
|
|
126
|
-
end
|
|
127
|
-
|
|
128
|
-
def create_complex_visitor
|
|
129
|
-
{
|
|
130
|
-
texts: 0,
|
|
131
|
-
links: 0,
|
|
132
|
-
images: 0,
|
|
133
|
-
headings: 0,
|
|
134
|
-
visit_text: proc { |_ctx, _text| 'continue' },
|
|
135
|
-
visit_heading: proc { |_ctx, _level, _text, _id| 'continue' },
|
|
136
|
-
visit_paragraph: proc { |_ctx, _text| 'continue' },
|
|
137
|
-
visit_link: proc { |_ctx, href, text, _title| ['custom', "[#{text}](#{href})"] },
|
|
138
|
-
visit_image: proc { |_ctx, _src, _alt, _title| 'skip' },
|
|
139
|
-
visit_strong: proc { |_ctx, _text| 'continue' },
|
|
140
|
-
visit_em: proc { |_ctx, _text| 'continue' },
|
|
141
|
-
visit_code: proc { |_ctx, _text| 'continue' },
|
|
142
|
-
visit_br: proc { |_ctx| 'continue' }
|
|
143
|
-
}
|
|
144
|
-
end
|
|
145
|
-
|
|
146
|
-
html = File.binread(fixture)
|
|
147
|
-
html.force_encoding(Encoding::UTF_8)
|
|
148
|
-
html.freeze
|
|
149
|
-
iterations = options[:iterations]
|
|
150
|
-
conversion_options = options[:format] == 'hocr' ? { hocr_spatial_tables: false } : {}
|
|
151
|
-
options_handle = if %w[convert-options inline-images-options metadata-options].include?(options[:scenario])
|
|
152
|
-
HtmlToMarkdown.options(conversion_options)
|
|
153
|
-
end
|
|
154
|
-
|
|
155
|
-
# Create visitor if specified
|
|
156
|
-
visitor = nil
|
|
157
|
-
if options[:visitor]
|
|
158
|
-
visitor_creators = {
|
|
159
|
-
'noop' => method(:create_noop_visitor),
|
|
160
|
-
'simple' => method(:create_simple_visitor),
|
|
161
|
-
'custom' => method(:create_custom_visitor),
|
|
162
|
-
'complex' => method(:create_complex_visitor)
|
|
163
|
-
}
|
|
164
|
-
creator = visitor_creators[options[:visitor]]
|
|
165
|
-
visitor = creator.call if creator
|
|
166
|
-
end
|
|
167
|
-
|
|
168
|
-
SCENARIO_RUNNERS = {
|
|
169
|
-
'convert-default' => ->(html, _options, _handle, _visitor) { HtmlToMarkdown.convert(html) },
|
|
170
|
-
'convert-options' => lambda do |html, _options, handle, _visitor|
|
|
171
|
-
raise ArgumentError, 'options handle required' unless handle
|
|
172
|
-
|
|
173
|
-
HtmlToMarkdown.convert_with_options(html, handle)
|
|
174
|
-
end,
|
|
175
|
-
'inline-images-default' => lambda { |html, _options, _handle, _visitor|
|
|
176
|
-
HtmlToMarkdown.convert_with_inline_images(html, nil, nil)
|
|
177
|
-
},
|
|
178
|
-
'inline-images-options' => lambda do |html, _options, handle, _visitor|
|
|
179
|
-
raise ArgumentError, 'options handle required' unless handle
|
|
180
|
-
|
|
181
|
-
HtmlToMarkdown.convert_with_inline_images_handle(html, handle, nil)
|
|
182
|
-
end,
|
|
183
|
-
'metadata-default' => ->(html, _options, _handle, _visitor) { HtmlToMarkdown.convert_with_metadata(html, nil, nil) },
|
|
184
|
-
'metadata-options' => lambda do |html, _options, handle, _visitor|
|
|
185
|
-
raise ArgumentError, 'options handle required' unless handle
|
|
186
|
-
|
|
187
|
-
HtmlToMarkdown.convert_with_metadata_handle(html, handle, nil)
|
|
188
|
-
end
|
|
189
|
-
}.freeze
|
|
190
|
-
|
|
191
|
-
def run_scenario(html, scenario, options, handle, visitor = nil)
|
|
192
|
-
if visitor
|
|
193
|
-
HtmlToMarkdown.convert_with_visitor(html, nil, visitor)
|
|
194
|
-
else
|
|
195
|
-
runner = SCENARIO_RUNNERS.fetch(scenario) { raise ArgumentError, "Unsupported scenario: #{scenario}" }
|
|
196
|
-
runner.call(html, options, handle, visitor)
|
|
197
|
-
end
|
|
198
|
-
end
|
|
199
|
-
|
|
200
|
-
run_scenario(html, options[:scenario], conversion_options, options_handle, visitor)
|
|
201
|
-
|
|
202
|
-
profile_output = ENV.fetch('HTML_TO_MARKDOWN_PROFILE_OUTPUT', nil)
|
|
203
|
-
if profile_output && HtmlToMarkdown.respond_to?(:start_profiling)
|
|
204
|
-
freq = Integer(ENV.fetch('HTML_TO_MARKDOWN_PROFILE_FREQUENCY', '1000'), 10)
|
|
205
|
-
HtmlToMarkdown.start_profiling(profile_output, freq)
|
|
206
|
-
end
|
|
207
|
-
|
|
208
|
-
start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
209
|
-
iterations.times { run_scenario(html, options[:scenario], conversion_options, options_handle, visitor) }
|
|
210
|
-
elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start
|
|
211
|
-
|
|
212
|
-
HtmlToMarkdown.stop_profiling if profile_output && HtmlToMarkdown.respond_to?(:stop_profiling)
|
|
213
|
-
|
|
214
|
-
payload_size_bytes = html.bytesize
|
|
215
|
-
bytes_processed = payload_size_bytes * iterations
|
|
216
|
-
ops_per_sec = iterations / elapsed
|
|
217
|
-
mb_per_sec = (bytes_processed.to_f / (1024 * 1024)) / elapsed
|
|
218
|
-
|
|
219
|
-
payload = %({
|
|
220
|
-
"language":"ruby",
|
|
221
|
-
"fixture":"#{json_escape(File.basename(fixture))}",
|
|
222
|
-
"fixture_path":"#{json_escape(fixture)}",
|
|
223
|
-
"scenario":"#{json_escape(options[:scenario])}",
|
|
224
|
-
"iterations":#{iterations},
|
|
225
|
-
"elapsed_seconds":#{format('%.8f', elapsed)},
|
|
226
|
-
"ops_per_sec":#{format('%.4f', ops_per_sec)},
|
|
227
|
-
"mb_per_sec":#{format('%.4f', mb_per_sec)},
|
|
228
|
-
"bytes_processed":#{bytes_processed},
|
|
229
|
-
"payload_size_bytes":#{payload_size_bytes}
|
|
230
|
-
})
|
|
231
|
-
|
|
232
|
-
puts payload.strip
|