kreuzberg 4.0.0.pre.rc.6 → 4.0.0.rc1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (175) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +0 -6
  3. data/.rubocop.yaml +534 -1
  4. data/Gemfile +2 -1
  5. data/Gemfile.lock +11 -11
  6. data/README.md +5 -10
  7. data/examples/async_patterns.rb +0 -1
  8. data/ext/kreuzberg_rb/extconf.rb +0 -10
  9. data/ext/kreuzberg_rb/native/Cargo.toml +15 -23
  10. data/ext/kreuzberg_rb/native/build.rs +2 -0
  11. data/ext/kreuzberg_rb/native/include/ieeefp.h +1 -1
  12. data/ext/kreuzberg_rb/native/include/msvc_compat/strings.h +1 -1
  13. data/ext/kreuzberg_rb/native/include/strings.h +2 -2
  14. data/ext/kreuzberg_rb/native/include/unistd.h +1 -1
  15. data/ext/kreuzberg_rb/native/src/lib.rs +16 -75
  16. data/kreuzberg.gemspec +14 -57
  17. data/lib/kreuzberg/cache_api.rb +0 -1
  18. data/lib/kreuzberg/cli.rb +2 -2
  19. data/lib/kreuzberg/config.rb +2 -9
  20. data/lib/kreuzberg/errors.rb +7 -75
  21. data/lib/kreuzberg/extraction_api.rb +0 -1
  22. data/lib/kreuzberg/setup_lib_path.rb +0 -1
  23. data/lib/kreuzberg/version.rb +1 -1
  24. data/lib/kreuzberg.rb +0 -21
  25. data/pkg/kreuzberg-4.0.0.rc1.gem +0 -0
  26. data/sig/kreuzberg.rbs +3 -55
  27. data/spec/binding/cli_proxy_spec.rb +4 -2
  28. data/spec/binding/cli_spec.rb +11 -12
  29. data/spec/examples.txt +104 -0
  30. data/spec/fixtures/config.yaml +1 -0
  31. data/spec/spec_helper.rb +1 -1
  32. data/vendor/kreuzberg/Cargo.toml +42 -112
  33. data/vendor/kreuzberg/README.md +2 -2
  34. data/vendor/kreuzberg/build.rs +4 -18
  35. data/vendor/kreuzberg/src/bin/profile_extract.rs +455 -0
  36. data/vendor/kreuzberg/src/cache/mod.rs +3 -27
  37. data/vendor/kreuzberg/src/core/batch_mode.rs +0 -60
  38. data/vendor/kreuzberg/src/core/extractor.rs +81 -202
  39. data/vendor/kreuzberg/src/core/io.rs +2 -4
  40. data/vendor/kreuzberg/src/core/mime.rs +12 -2
  41. data/vendor/kreuzberg/src/core/mod.rs +1 -4
  42. data/vendor/kreuzberg/src/core/pipeline.rs +33 -111
  43. data/vendor/kreuzberg/src/embeddings.rs +16 -125
  44. data/vendor/kreuzberg/src/error.rs +1 -1
  45. data/vendor/kreuzberg/src/extraction/docx.rs +1 -1
  46. data/vendor/kreuzberg/src/extraction/image.rs +13 -13
  47. data/vendor/kreuzberg/src/extraction/libreoffice.rs +1 -0
  48. data/vendor/kreuzberg/src/extraction/mod.rs +5 -9
  49. data/vendor/kreuzberg/src/extraction/office_metadata/mod.rs +0 -2
  50. data/vendor/kreuzberg/src/extraction/pandoc/batch.rs +275 -0
  51. data/vendor/kreuzberg/src/extraction/pandoc/mime_types.rs +178 -0
  52. data/vendor/kreuzberg/src/extraction/pandoc/mod.rs +491 -0
  53. data/vendor/kreuzberg/src/extraction/pandoc/server.rs +496 -0
  54. data/vendor/kreuzberg/src/extraction/pandoc/subprocess.rs +1188 -0
  55. data/vendor/kreuzberg/src/extraction/pandoc/version.rs +162 -0
  56. data/vendor/kreuzberg/src/extractors/archive.rs +0 -21
  57. data/vendor/kreuzberg/src/extractors/docx.rs +128 -16
  58. data/vendor/kreuzberg/src/extractors/email.rs +0 -14
  59. data/vendor/kreuzberg/src/extractors/excel.rs +20 -19
  60. data/vendor/kreuzberg/src/extractors/html.rs +154 -137
  61. data/vendor/kreuzberg/src/extractors/image.rs +4 -7
  62. data/vendor/kreuzberg/src/extractors/mod.rs +9 -106
  63. data/vendor/kreuzberg/src/extractors/pandoc.rs +201 -0
  64. data/vendor/kreuzberg/src/extractors/pdf.rs +15 -12
  65. data/vendor/kreuzberg/src/extractors/pptx.rs +3 -17
  66. data/vendor/kreuzberg/src/extractors/structured.rs +0 -14
  67. data/vendor/kreuzberg/src/extractors/text.rs +5 -23
  68. data/vendor/kreuzberg/src/extractors/xml.rs +0 -7
  69. data/vendor/kreuzberg/src/keywords/rake.rs +1 -0
  70. data/vendor/kreuzberg/src/lib.rs +1 -4
  71. data/vendor/kreuzberg/src/mcp/mod.rs +1 -1
  72. data/vendor/kreuzberg/src/mcp/server.rs +3 -5
  73. data/vendor/kreuzberg/src/ocr/processor.rs +2 -18
  74. data/vendor/kreuzberg/src/pdf/error.rs +1 -1
  75. data/vendor/kreuzberg/src/pdf/table.rs +44 -17
  76. data/vendor/kreuzberg/src/pdf/text.rs +3 -0
  77. data/vendor/kreuzberg/src/plugins/extractor.rs +5 -8
  78. data/vendor/kreuzberg/src/plugins/ocr.rs +11 -2
  79. data/vendor/kreuzberg/src/plugins/processor.rs +1 -2
  80. data/vendor/kreuzberg/src/plugins/registry.rs +0 -13
  81. data/vendor/kreuzberg/src/plugins/validator.rs +8 -9
  82. data/vendor/kreuzberg/src/stopwords/mod.rs +2 -2
  83. data/vendor/kreuzberg/src/types.rs +12 -42
  84. data/vendor/kreuzberg/tests/batch_orchestration.rs +5 -19
  85. data/vendor/kreuzberg/tests/batch_processing.rs +3 -15
  86. data/vendor/kreuzberg/tests/chunking_offset_demo.rs +92 -0
  87. data/vendor/kreuzberg/tests/concurrency_stress.rs +1 -17
  88. data/vendor/kreuzberg/tests/config_features.rs +0 -18
  89. data/vendor/kreuzberg/tests/config_loading_tests.rs +39 -15
  90. data/vendor/kreuzberg/tests/core_integration.rs +7 -24
  91. data/vendor/kreuzberg/tests/csv_integration.rs +81 -71
  92. data/vendor/kreuzberg/tests/docx_metadata_extraction_test.rs +25 -23
  93. data/vendor/kreuzberg/tests/pandoc_integration.rs +503 -0
  94. data/vendor/kreuzberg/tests/pipeline_integration.rs +1 -0
  95. data/vendor/kreuzberg/tests/plugin_postprocessor_test.rs +1 -0
  96. data/vendor/kreuzberg/tests/registry_integration_tests.rs +22 -1
  97. data/vendor/kreuzberg/tests/security_validation.rs +1 -12
  98. metadata +25 -90
  99. data/.rubocop.yml +0 -538
  100. data/ext/kreuzberg_rb/native/Cargo.lock +0 -6535
  101. data/lib/kreuzberg/error_context.rb +0 -32
  102. data/vendor/kreuzberg/benches/otel_overhead.rs +0 -48
  103. data/vendor/kreuzberg/src/extraction/markdown.rs +0 -213
  104. data/vendor/kreuzberg/src/extraction/office_metadata/odt_properties.rs +0 -287
  105. data/vendor/kreuzberg/src/extractors/bibtex.rs +0 -469
  106. data/vendor/kreuzberg/src/extractors/docbook.rs +0 -502
  107. data/vendor/kreuzberg/src/extractors/epub.rs +0 -707
  108. data/vendor/kreuzberg/src/extractors/fictionbook.rs +0 -491
  109. data/vendor/kreuzberg/src/extractors/fictionbook.rs.backup2 +0 -738
  110. data/vendor/kreuzberg/src/extractors/jats.rs +0 -1051
  111. data/vendor/kreuzberg/src/extractors/jupyter.rs +0 -367
  112. data/vendor/kreuzberg/src/extractors/latex.rs +0 -652
  113. data/vendor/kreuzberg/src/extractors/markdown.rs +0 -700
  114. data/vendor/kreuzberg/src/extractors/odt.rs +0 -628
  115. data/vendor/kreuzberg/src/extractors/opml.rs +0 -634
  116. data/vendor/kreuzberg/src/extractors/orgmode.rs +0 -528
  117. data/vendor/kreuzberg/src/extractors/rst.rs +0 -576
  118. data/vendor/kreuzberg/src/extractors/rtf.rs +0 -810
  119. data/vendor/kreuzberg/src/extractors/security.rs +0 -484
  120. data/vendor/kreuzberg/src/extractors/security_tests.rs +0 -367
  121. data/vendor/kreuzberg/src/extractors/typst.rs +0 -650
  122. data/vendor/kreuzberg/src/panic_context.rs +0 -154
  123. data/vendor/kreuzberg/tests/api_extract_multipart.rs +0 -52
  124. data/vendor/kreuzberg/tests/bibtex_parity_test.rs +0 -421
  125. data/vendor/kreuzberg/tests/docbook_extractor_tests.rs +0 -498
  126. data/vendor/kreuzberg/tests/docx_vs_pandoc_comparison.rs +0 -370
  127. data/vendor/kreuzberg/tests/epub_native_extractor_tests.rs +0 -275
  128. data/vendor/kreuzberg/tests/fictionbook_extractor_tests.rs +0 -228
  129. data/vendor/kreuzberg/tests/html_table_test.rs +0 -551
  130. data/vendor/kreuzberg/tests/instrumentation_test.rs +0 -139
  131. data/vendor/kreuzberg/tests/jats_extractor_tests.rs +0 -639
  132. data/vendor/kreuzberg/tests/jupyter_extractor_tests.rs +0 -704
  133. data/vendor/kreuzberg/tests/latex_extractor_tests.rs +0 -496
  134. data/vendor/kreuzberg/tests/markdown_extractor_tests.rs +0 -490
  135. data/vendor/kreuzberg/tests/odt_extractor_tests.rs +0 -695
  136. data/vendor/kreuzberg/tests/opml_extractor_tests.rs +0 -616
  137. data/vendor/kreuzberg/tests/orgmode_extractor_tests.rs +0 -822
  138. data/vendor/kreuzberg/tests/rst_extractor_tests.rs +0 -692
  139. data/vendor/kreuzberg/tests/rtf_extractor_tests.rs +0 -776
  140. data/vendor/kreuzberg/tests/typst_behavioral_tests.rs +0 -1259
  141. data/vendor/kreuzberg/tests/typst_extractor_tests.rs +0 -647
  142. data/vendor/rb-sys/.cargo-ok +0 -1
  143. data/vendor/rb-sys/.cargo_vcs_info.json +0 -6
  144. data/vendor/rb-sys/Cargo.lock +0 -393
  145. data/vendor/rb-sys/Cargo.toml +0 -70
  146. data/vendor/rb-sys/Cargo.toml.orig +0 -57
  147. data/vendor/rb-sys/LICENSE-APACHE +0 -190
  148. data/vendor/rb-sys/LICENSE-MIT +0 -21
  149. data/vendor/rb-sys/bin/release.sh +0 -21
  150. data/vendor/rb-sys/build/features.rs +0 -108
  151. data/vendor/rb-sys/build/main.rs +0 -246
  152. data/vendor/rb-sys/build/stable_api_config.rs +0 -153
  153. data/vendor/rb-sys/build/version.rs +0 -48
  154. data/vendor/rb-sys/readme.md +0 -36
  155. data/vendor/rb-sys/src/bindings.rs +0 -21
  156. data/vendor/rb-sys/src/hidden.rs +0 -11
  157. data/vendor/rb-sys/src/lib.rs +0 -34
  158. data/vendor/rb-sys/src/macros.rs +0 -371
  159. data/vendor/rb-sys/src/memory.rs +0 -53
  160. data/vendor/rb-sys/src/ruby_abi_version.rs +0 -38
  161. data/vendor/rb-sys/src/special_consts.rs +0 -31
  162. data/vendor/rb-sys/src/stable_api/compiled.c +0 -179
  163. data/vendor/rb-sys/src/stable_api/compiled.rs +0 -257
  164. data/vendor/rb-sys/src/stable_api/ruby_2_6.rs +0 -316
  165. data/vendor/rb-sys/src/stable_api/ruby_2_7.rs +0 -316
  166. data/vendor/rb-sys/src/stable_api/ruby_3_0.rs +0 -324
  167. data/vendor/rb-sys/src/stable_api/ruby_3_1.rs +0 -317
  168. data/vendor/rb-sys/src/stable_api/ruby_3_2.rs +0 -315
  169. data/vendor/rb-sys/src/stable_api/ruby_3_3.rs +0 -326
  170. data/vendor/rb-sys/src/stable_api/ruby_3_4.rs +0 -327
  171. data/vendor/rb-sys/src/stable_api.rs +0 -261
  172. data/vendor/rb-sys/src/symbol.rs +0 -31
  173. data/vendor/rb-sys/src/tracking_allocator.rs +0 -332
  174. data/vendor/rb-sys/src/utils.rs +0 -89
  175. data/vendor/rb-sys/src/value_type.rs +0 -7
data/vendor/kreuzberg/tests/jats_extractor_tests.rs DELETED
@@ -1,639 +0,0 @@
1
- //! Comprehensive test suite for the JATS (Journal Article Tag Suite) extractor.
2
- //!
3
- //! This test suite validates all aspects of the JATS extractor including:
4
- //! - Metadata extraction (title, authors, affiliations, DOI, keywords, dates)
5
- //! - Article content extraction (sections, paragraphs)
6
- //! - Table extraction with proper structure
7
- //! - Citation handling
8
- //! - Edge cases and error handling
9
-
10
- #[cfg(all(test, feature = "xml"))]
11
- mod jats_extractor_tests {
12
- use kreuzberg::core::config::ExtractionConfig;
13
- use kreuzberg::extractors::JatsExtractor;
14
- use kreuzberg::plugins::{DocumentExtractor, Plugin};
15
- use std::path::PathBuf;
16
-
17
- fn jats_fixture(name: &str) -> PathBuf {
18
- PathBuf::from(env!("CARGO_MANIFEST_DIR"))
19
- .join("../../test_documents/jats")
20
- .join(name)
21
- }
22
-
23
- /// Test basic JATS article extraction with all key metadata fields
24
- #[tokio::test]
25
- async fn test_extract_complete_jats_article() {
26
- let jats_content = r#"<?xml version="1.0" encoding="UTF-8"?>
27
- <article xmlns:xlink="http://www.w3.org/1999/xlink" article-type="research-article">
28
- <front>
29
- <article-meta>
30
- <article-title>Effects of Caffeine on Human Health</article-title>
31
- <subtitle>A Systematic Review and Meta-Analysis</subtitle>
32
- <contrib-group>
33
- <contrib contrib-type="author">
34
- <name>
35
- <surname>Smith</surname>
36
- <given-names>John A.</given-names>
37
- </name>
38
- </contrib>
39
- <contrib contrib-type="author">
40
- <name>
41
- <surname>Johnson</surname>
42
- <given-names>Jane B.</given-names>
43
- </name>
44
- </contrib>
45
- </contrib-group>
46
- <aff id="aff1">Department of Medicine, Harvard University, Cambridge, MA</aff>
47
- <article-id pub-id-type="doi">10.1371/journal.pmed.0020124</article-id>
48
- <article-id pub-id-type="pii">05-PLME-RA-0071R2</article-id>
49
- <pub-date pub-type="epub">
50
- <day>18</day>
51
- <month>04</month>
52
- <year>2005</year>
53
- </pub-date>
54
- <volume>2</volume>
55
- <issue>4</issue>
56
- <fpage>e124</fpage>
57
- <lpage>e132</lpage>
58
- <kwd-group>
59
- <kwd>caffeine</kwd>
60
- <kwd>meta-analysis</kwd>
61
- <kwd>systematic review</kwd>
62
- </kwd-group>
63
- <abstract>
64
- <sec>
65
- <title>Background</title>
66
- <p>Caffeine is one of the most widely consumed psychoactive substances.</p>
67
- </sec>
68
- </abstract>
69
- </article-meta>
70
- </front>
71
- <body>
72
- <sec id="s1">
73
- <title>Introduction</title>
74
- <p>This review examines the evidence for effects of caffeine.</p>
75
- </sec>
76
- </body>
77
- </article>"#;
78
-
79
- let extractor = JatsExtractor::new();
80
- let config = ExtractionConfig::default();
81
- let result = extractor
82
- .extract_bytes(jats_content.as_bytes(), "application/x-jats+xml", &config)
83
- .await;
84
-
85
- assert!(result.is_ok());
86
- let extraction = result.unwrap();
87
-
88
- assert!(extraction.content.contains("Effects of Caffeine"));
89
- assert!(extraction.content.contains("Introduction"));
90
-
91
- assert!(extraction.metadata.subject.is_some());
92
- let subject = extraction.metadata.subject.unwrap();
93
- assert!(subject.contains("Effects of Caffeine"));
94
-
95
- assert!(subject.contains("10.1371"));
96
-
97
- assert!(subject.contains("caffeine") || subject.contains("Keywords"));
98
- }
99
-
100
- /// Test extraction of rich metadata including all author and affiliation data
101
- #[tokio::test]
102
- async fn test_extract_rich_author_affiliation_metadata() {
103
- let jats_content = r#"<?xml version="1.0" encoding="UTF-8"?>
104
- <article>
105
- <front>
106
- <article-meta>
107
- <article-title>Advanced Study</article-title>
108
- <contrib-group>
109
- <contrib contrib-type="author">
110
- <name>
111
- <surname>Alpha</surname>
112
- <given-names>First</given-names>
113
- </name>
114
- <xref ref-type="aff" rid="aff1">1</xref>
115
- <xref ref-type="aff" rid="aff2">2</xref>
116
- </contrib>
117
- <contrib contrib-type="author">
118
- <name>
119
- <surname>Beta</surname>
120
- <given-names>Second</given-names>
121
- </name>
122
- <xref ref-type="aff" rid="aff1">1</xref>
123
- </contrib>
124
- <contrib contrib-type="author">
125
- <name>
126
- <surname>Gamma</surname>
127
- <given-names>Third</given-names>
128
- </name>
129
- <xref ref-type="aff" rid="aff3">3</xref>
130
- <role>Correspondence</role>
131
- </contrib>
132
- </contrib-group>
133
- <aff id="aff1"><label>1</label>Department of Science, University A, City A</aff>
134
- <aff id="aff2"><label>2</label>Research Institute, City B</aff>
135
- <aff id="aff3"><label>3</label>Medical Center, City C</aff>
136
- </article-meta>
137
- </front>
138
- </article>"#;
139
-
140
- let extractor = JatsExtractor::new();
141
- let config = ExtractionConfig::default();
142
- let result = extractor
143
- .extract_bytes(jats_content.as_bytes(), "application/x-jats+xml", &config)
144
- .await;
145
-
146
- assert!(result.is_ok());
147
- let extraction = result.unwrap();
148
-
149
- let subject = extraction.metadata.subject.unwrap();
150
- assert!(subject.contains("Alpha"));
151
- assert!(subject.contains("Beta"));
152
- assert!(subject.contains("Gamma"));
153
- assert!(subject.contains("Department of Science"));
154
- }
155
-
156
- /// Test section hierarchy extraction in article body
157
- #[tokio::test]
158
- async fn test_extract_section_hierarchy() {
159
- let jats_content = r#"<?xml version="1.0" encoding="UTF-8"?>
160
- <article>
161
- <front>
162
- <article-meta>
163
- <article-title>Research Article</article-title>
164
- </article-meta>
165
- </front>
166
- <body>
167
- <sec id="s1">
168
- <title>Introduction</title>
169
- <p>Introduction content here.</p>
170
- </sec>
171
- <sec id="s2">
172
- <title>Methods</title>
173
- <sec id="s2a">
174
- <title>Study Design</title>
175
- <p>Design content here.</p>
176
- </sec>
177
- <sec id="s2b">
178
- <title>Participants</title>
179
- <p>Participant content here.</p>
180
- </sec>
181
- </sec>
182
- <sec id="s3">
183
- <title>Results</title>
184
- <p>Results content here.</p>
185
- </sec>
186
- <sec id="s4">
187
- <title>Discussion</title>
188
- <p>Discussion content here.</p>
189
- </sec>
190
- <sec id="s5">
191
- <title>Conclusions</title>
192
- <p>Conclusion content here.</p>
193
- </sec>
194
- </body>
195
- </article>"#;
196
-
197
- let extractor = JatsExtractor::new();
198
- let config = ExtractionConfig::default();
199
- let result = extractor
200
- .extract_bytes(jats_content.as_bytes(), "application/x-jats+xml", &config)
201
- .await;
202
-
203
- assert!(result.is_ok());
204
- let extraction = result.unwrap();
205
-
206
- assert!(extraction.content.contains("Introduction"));
207
- assert!(extraction.content.contains("Methods"));
208
- assert!(extraction.content.contains("Results"));
209
- assert!(extraction.content.contains("Discussion"));
210
- assert!(extraction.content.contains("Conclusions"));
211
- assert!(extraction.content.contains("Study Design"));
212
- assert!(extraction.content.contains("Participants"));
213
- }
214
-
215
- /// Test table extraction with headers and data rows
216
- #[tokio::test]
217
- async fn test_extract_tables_with_captions() {
218
- let jats_content = r#"<?xml version="1.0" encoding="UTF-8"?>
219
- <article>
220
- <front>
221
- <article-meta>
222
- <article-title>Study Results</article-title>
223
- </article-meta>
224
- </front>
225
- <body>
226
- <sec id="s1">
227
- <title>Results</title>
228
- <table-wrap id="tbl1">
229
- <label>Table 1</label>
230
- <caption>
231
- <title>Characteristics of Study Population</title>
232
- <p>Baseline characteristics of enrolled subjects.</p>
233
- </caption>
234
- <table>
235
- <thead>
236
- <tr>
237
- <th>Parameter</th>
238
- <th>Group A (n=50)</th>
239
- <th>Group B (n=50)</th>
240
- <th>P-value</th>
241
- </tr>
242
- </thead>
243
- <tbody>
244
- <tr>
245
- <td>Age (years)</td>
246
- <td>45.3 ± 8.2</td>
247
- <td>44.9 ± 7.8</td>
248
- <td>0.45</td>
249
- </tr>
250
- <tr>
251
- <td>Sex (M/F)</td>
252
- <td>28/22</td>
253
- <td>26/24</td>
254
- <td>0.58</td>
255
- </tr>
256
- <tr>
257
- <td>BMI (kg/m²)</td>
258
- <td>25.1 ± 3.2</td>
259
- <td>24.8 ± 2.9</td>
260
- <td>0.62</td>
261
- </tr>
262
- </tbody>
263
- </table>
264
- </table-wrap>
265
- </sec>
266
- </body>
267
- </article>"#;
268
-
269
- let extractor = JatsExtractor::new();
270
- let config = ExtractionConfig::default();
271
- let result = extractor
272
- .extract_bytes(jats_content.as_bytes(), "application/x-jats+xml", &config)
273
- .await;
274
-
275
- assert!(result.is_ok());
276
- let extraction = result.unwrap();
277
-
278
- assert_eq!(extraction.tables.len(), 1);
279
- let table = &extraction.tables[0];
280
-
281
- assert!(table.cells.len() >= 3);
282
- assert_eq!(table.cells[0].len(), 4);
283
-
284
- assert!(table.cells[0][0].contains("Parameter"));
285
- assert!(table.cells[1][0].contains("Age"));
286
- assert!(table.cells[2][0].contains("Sex"));
287
- }
288
-
289
- /// Test multiple tables extraction
290
- #[tokio::test]
291
- async fn test_extract_multiple_tables() {
292
- let jats_content = r#"<?xml version="1.0" encoding="UTF-8"?>
293
- <article>
294
- <front>
295
- <article-meta>
296
- <article-title>Multi-Table Article</article-title>
297
- </article-meta>
298
- </front>
299
- <body>
300
- <table-wrap id="tbl1">
301
- <table>
302
- <thead>
303
- <tr><th>A</th><th>B</th></tr>
304
- </thead>
305
- <tbody>
306
- <tr><td>1</td><td>2</td></tr>
307
- </tbody>
308
- </table>
309
- </table-wrap>
310
- <table-wrap id="tbl2">
311
- <table>
312
- <thead>
313
- <tr><th>X</th><th>Y</th><th>Z</th></tr>
314
- </thead>
315
- <tbody>
316
- <tr><td>a</td><td>b</td><td>c</td></tr>
317
- </tbody>
318
- </table>
319
- </table-wrap>
320
- </body>
321
- </article>"#;
322
-
323
- let extractor = JatsExtractor::new();
324
- let config = ExtractionConfig::default();
325
- let result = extractor
326
- .extract_bytes(jats_content.as_bytes(), "application/x-jats+xml", &config)
327
- .await;
328
-
329
- assert!(result.is_ok());
330
- let extraction = result.unwrap();
331
-
332
- assert_eq!(extraction.tables.len(), 2);
333
- assert_eq!(extraction.tables[0].cells[0].len(), 2);
334
- assert_eq!(extraction.tables[1].cells[0].len(), 3);
335
- }
336
-
337
- /// Test citation extraction in text with xref elements
338
- #[tokio::test]
339
- async fn test_extract_citations_in_text() {
340
- let jats_content = r#"<?xml version="1.0" encoding="UTF-8"?>
341
- <article>
342
- <front>
343
- <article-meta>
344
- <article-title>Citation Study</article-title>
345
- </article-meta>
346
- </front>
347
- <body>
348
- <sec id="s1">
349
- <title>Introduction</title>
350
- <p>Previous research has shown effectiveness <xref ref-type="bibr" rid="ref1">1</xref>.
351
- Other studies confirm this finding <xref ref-type="bibr" rid="ref2">2</xref>.</p>
352
- </sec>
353
- </body>
354
- <back>
355
- <ref-list>
356
- <ref id="ref1">
357
- <element-citation publication-type="journal">
358
- <person-group person-group-type="author">
359
- <name>
360
- <surname>Author</surname>
361
- <given-names>First</given-names>
362
- </name>
363
- </person-group>
364
- <article-title>Original Research</article-title>
365
- <source>Journal Name</source>
366
- <year>2020</year>
367
- </element-citation>
368
- </ref>
369
- <ref id="ref2">
370
- <element-citation publication-type="journal">
371
- <person-group person-group-type="author">
372
- <name>
373
- <surname>Researcher</surname>
374
- <given-names>Second</given-names>
375
- </name>
376
- </person-group>
377
- <article-title>Confirmatory Study</article-title>
378
- <source>Other Journal</source>
379
- <year>2021</year>
380
- </element-citation>
381
- </ref>
382
- </ref-list>
383
- </back>
384
- </article>"#;
385
-
386
- let extractor = JatsExtractor::new();
387
- let config = ExtractionConfig::default();
388
- let result = extractor
389
- .extract_bytes(jats_content.as_bytes(), "application/x-jats+xml", &config)
390
- .await;
391
-
392
- assert!(result.is_ok());
393
- let extraction = result.unwrap();
394
-
395
- assert!(extraction.content.contains("Previous research"));
396
- assert!(extraction.content.contains("Other studies"));
397
- }
398
-
399
- /// Test abstract extraction with structured sections
400
- #[tokio::test]
401
- async fn test_extract_structured_abstract() {
402
- let jats_content = r#"<?xml version="1.0" encoding="UTF-8"?>
403
- <article>
404
- <front>
405
- <article-meta>
406
- <article-title>Abstract Study</article-title>
407
- <abstract>
408
- <sec>
409
- <title>Background</title>
410
- <p>This is the background information of the study.</p>
411
- </sec>
412
- <sec>
413
- <title>Methods and Findings</title>
414
- <p>We used quantitative analysis to evaluate the hypothesis.</p>
415
- </sec>
416
- <sec>
417
- <title>Conclusions</title>
418
- <p>The study provides evidence that the hypothesis is correct.</p>
419
- </sec>
420
- </abstract>
421
- </article-meta>
422
- </front>
423
- </article>"#;
424
-
425
- let extractor = JatsExtractor::new();
426
- let config = ExtractionConfig::default();
427
- let result = extractor
428
- .extract_bytes(jats_content.as_bytes(), "application/x-jats+xml", &config)
429
- .await;
430
-
431
- assert!(result.is_ok());
432
- let extraction = result.unwrap();
433
-
434
- let subject = extraction.metadata.subject.unwrap();
435
- assert!(subject.contains("background") || subject.contains("Background") || subject.contains("Abstract"));
436
- }
437
-
438
- /// Test corresponding author extraction
439
- #[tokio::test]
440
- async fn test_extract_corresponding_author() {
441
- let jats_content = r#"<?xml version="1.0" encoding="UTF-8"?>
442
- <article>
443
- <front>
444
- <article-meta>
445
- <article-title>Study</article-title>
446
- <author-notes>
447
- <corresp id="cor1"><label>*</label>Corresponding author. E-mail: john.smith@example.com</corresp>
448
- </author-notes>
449
- </article-meta>
450
- </front>
451
- </article>"#;
452
-
453
- let extractor = JatsExtractor::new();
454
- let config = ExtractionConfig::default();
455
- let result = extractor
456
- .extract_bytes(jats_content.as_bytes(), "application/x-jats+xml", &config)
457
- .await;
458
-
459
- assert!(result.is_ok());
460
- let extraction = result.unwrap();
461
-
462
- assert!(extraction.metadata.subject.is_some());
463
- }
464
-
465
- /// Test publication date extraction in various formats
466
- #[tokio::test]
467
- async fn test_extract_publication_date() {
468
- let jats_content = r#"<?xml version="1.0" encoding="UTF-8"?>
469
- <article>
470
- <front>
471
- <article-meta>
472
- <article-title>Publication Test</article-title>
473
- <pub-date pub-type="epub">
474
- <day>15</day>
475
- <month>06</month>
476
- <year>2023</year>
477
- </pub-date>
478
- </article-meta>
479
- </front>
480
- </article>"#;
481
-
482
- let extractor = JatsExtractor::new();
483
- let config = ExtractionConfig::default();
484
- let result = extractor
485
- .extract_bytes(jats_content.as_bytes(), "application/x-jats+xml", &config)
486
- .await;
487
-
488
- assert!(result.is_ok());
489
- let extraction = result.unwrap();
490
-
491
- assert!(extraction.metadata.date.is_some());
492
- }
493
-
494
- /// Test handling of empty/minimal JATS documents
495
- #[tokio::test]
496
- async fn test_extract_minimal_jats() {
497
- let jats_content = r#"<?xml version="1.0" encoding="UTF-8"?>
498
- <article>
499
- <front>
500
- <article-meta>
501
- </article-meta>
502
- </front>
503
- <body>
504
- </body>
505
- </article>"#;
506
-
507
- let extractor = JatsExtractor::new();
508
- let config = ExtractionConfig::default();
509
- let result = extractor
510
- .extract_bytes(jats_content.as_bytes(), "application/x-jats+xml", &config)
511
- .await;
512
-
513
- assert!(result.is_ok());
514
- let extraction = result.unwrap();
515
- assert!(extraction.content.is_empty() || extraction.content.trim().is_empty());
516
- }
517
-
518
- /// Test MIME type support
519
- #[test]
520
- fn test_jats_supported_mime_types() {
521
- let extractor = JatsExtractor::new();
522
- let mime_types = extractor.supported_mime_types();
523
-
524
- assert!(mime_types.contains(&"application/x-jats+xml"));
525
- assert!(mime_types.contains(&"text/jats"));
526
- }
527
-
528
- /// Test extractor priority value
529
- #[test]
530
- fn test_jats_extractor_priority() {
531
- let extractor = JatsExtractor::new();
532
- assert_eq!(extractor.priority(), 50);
533
- }
534
-
535
- /// Test plugin interface compliance
536
- #[test]
537
- fn test_jats_plugin_interface() {
538
- let extractor = JatsExtractor::new();
539
- assert_eq!(extractor.name(), "jats-extractor");
540
- assert!(!extractor.version().is_empty());
541
- assert!(extractor.initialize().is_ok());
542
- assert!(extractor.shutdown().is_ok());
543
- }
544
-
545
- /// Test mixed content with tables and paragraphs
546
- #[tokio::test]
547
- async fn test_extract_mixed_content() {
548
- let jats_content = r#"<?xml version="1.0" encoding="UTF-8"?>
549
- <article>
550
- <front>
551
- <article-meta>
552
- <article-title>Mixed Content</article-title>
553
- </article-meta>
554
- </front>
555
- <body>
556
- <sec id="s1">
557
- <title>Analysis</title>
558
- <p>First paragraph with data.</p>
559
- <table-wrap id="tbl1">
560
- <table>
561
- <thead>
562
- <tr><th>Data</th><th>Value</th></tr>
563
- </thead>
564
- <tbody>
565
- <tr><td>Sample</td><td>100</td></tr>
566
- </tbody>
567
- </table>
568
- </table-wrap>
569
- <p>Second paragraph after table.</p>
570
- </sec>
571
- </body>
572
- </article>"#;
573
-
574
- let extractor = JatsExtractor::new();
575
- let config = ExtractionConfig::default();
576
- let result = extractor
577
- .extract_bytes(jats_content.as_bytes(), "application/x-jats+xml", &config)
578
- .await;
579
-
580
- assert!(result.is_ok());
581
- let extraction = result.unwrap();
582
-
583
- assert!(extraction.content.contains("First paragraph"));
584
- assert!(extraction.content.contains("Second paragraph"));
585
- assert_eq!(extraction.tables.len(), 1);
586
- }
587
-
588
- /// Test extraction with multiple keyword groups
589
- #[tokio::test]
590
- async fn test_extract_multiple_keywords() {
591
- let jats_content = r#"<?xml version="1.0" encoding="UTF-8"?>
592
- <article>
593
- <front>
594
- <article-meta>
595
- <article-title>Keyword Test</article-title>
596
- <kwd-group xml:lang="en">
597
- <kwd>primary keyword</kwd>
598
- <kwd>secondary keyword</kwd>
599
- </kwd-group>
600
- <kwd-group xml:lang="es">
601
- <kwd>palabra clave</kwd>
602
- </kwd-group>
603
- </article-meta>
604
- </front>
605
- </article>"#;
606
-
607
- let extractor = JatsExtractor::new();
608
- let config = ExtractionConfig::default();
609
- let result = extractor
610
- .extract_bytes(jats_content.as_bytes(), "application/x-jats+xml", &config)
611
- .await;
612
-
613
- assert!(result.is_ok());
614
- let extraction = result.unwrap();
615
-
616
- let subject = extraction.metadata.subject.unwrap();
617
- assert!(subject.contains("keyword") || subject.contains("Keyword"));
618
- }
619
-
620
- /// Test full extraction workflow on file
621
- #[tokio::test]
622
- async fn test_extract_jats_file() {
623
- let extractor = JatsExtractor::new();
624
- let config = ExtractionConfig::default();
625
-
626
- let test_file = jats_fixture("sample_article.jats");
627
- if test_file.exists() {
628
- let result = extractor
629
- .extract_file(&test_file, "application/x-jats+xml", &config)
630
- .await;
631
-
632
- assert!(result.is_ok());
633
- let extraction = result.unwrap();
634
-
635
- assert!(!extraction.content.is_empty());
636
- assert!(extraction.metadata.subject.is_some());
637
- }
638
- }
639
- }