relaton 3.1.5 → 3.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.adoc +1 -0
- data/lib/relaton/arxiv/README.adoc +59 -0
- data/lib/relaton/arxiv/api.rb +42 -0
- data/lib/relaton/arxiv/bibliography.rb +45 -0
- data/lib/relaton/arxiv/parser.rb +167 -0
- data/lib/relaton/arxiv/processor.rb +66 -0
- data/lib/relaton/arxiv/util.rb +8 -0
- data/lib/relaton/arxiv.rb +23 -0
- data/lib/relaton/bibcollection.rb +45 -83
- data/lib/relaton/cie/data_fetcher.rb +142 -13
- data/lib/relaton/cli/command.rb +7 -3
- data/lib/relaton/cli/relaton_file.rb +101 -64
- data/lib/relaton/cli/subcommand_collection.rb +6 -24
- data/lib/relaton/cli/xml_convertor.rb +6 -5
- data/lib/relaton/cli/xml_to_html_renderer.rb +1 -2
- data/lib/relaton/cli.rb +79 -20
- data/lib/relaton/collection_xml.rb +145 -0
- data/lib/relaton/db/registry.rb +6 -2
- data/lib/relaton/doi/README.adoc +14 -3
- data/lib/relaton/doi/crossref.rb +2 -12
- data/lib/relaton/doi/datacite.rb +40 -0
- data/lib/relaton/doi/datacite_parser.rb +301 -0
- data/lib/relaton/doi/http.rb +86 -0
- data/lib/relaton/doi/processor.rb +1 -1
- data/lib/relaton/doi/registration_agency.rb +30 -0
- data/lib/relaton/doi.rb +45 -0
- data/lib/relaton/iho/README.adoc +14 -0
- data/lib/relaton/iho/bibliography.rb +43 -1
- data/lib/relaton/iho/processor.rb +11 -0
- data/lib/relaton/iso/bibliography.rb +49 -2
- data/lib/relaton/iso/processor.rb +12 -0
- data/lib/relaton/version.rb +1 -1
- data/lib/relaton-cli.rb +0 -1
- data/lib/relaton.rb +1 -0
- metadata +16 -5
- data/lib/relaton/element_finder.rb +0 -29
|
@@ -87,6 +87,15 @@ module Relaton
|
|
|
87
87
|
# Cloudflare; tune via RELATON_CIE_CONCURRENCY (see .concurrency).
|
|
88
88
|
DEFAULT_CONCURRENCY = 5
|
|
89
89
|
|
|
90
|
+
# Languages a catalogue translation can carry (relaton-cie#13). A hit for
|
|
91
|
+
# one of them (`cie-232-2019-de?product_id=…`, `CIE 232:2019(DE)`) has the
|
|
92
|
+
# primary id of its English document, so it merges into that record.
|
|
93
|
+
TRANSLATION_LANGUAGES = %w[de fr es it ru ja zh pt ko].freeze
|
|
94
|
+
|
|
95
|
+
# ISO 15924 script of a translation language that is not Latin.
|
|
96
|
+
SCRIPTS = { "ru" => "Cyrl", "ja" => "Jpan", "zh" => "Hans",
|
|
97
|
+
"ko" => "Kore" }.freeze
|
|
98
|
+
|
|
90
99
|
# Per-worker pacing bounds (seconds). Each worker starts at BASE_GAP and
|
|
91
100
|
# doubles its own gap up to MAX_GAP on trouble (see Pacing / #time_req).
|
|
92
101
|
BASE_GAP = 1
|
|
@@ -99,7 +108,7 @@ module Relaton
|
|
|
99
108
|
[(ENV["RELATON_CIE_CONCURRENCY"] || DEFAULT_CONCURRENCY).to_i, 1].max
|
|
100
109
|
end
|
|
101
110
|
|
|
102
|
-
def initialize(output, format)
|
|
111
|
+
def initialize(output, format) # rubocop:disable Metrics/MethodLength
|
|
103
112
|
super
|
|
104
113
|
# Guards the shared bookkeeping (@files, @errors, @seen) and the index
|
|
105
114
|
# mutation while workers fetch detail pages in parallel. The slow
|
|
@@ -109,6 +118,10 @@ module Relaton
|
|
|
109
118
|
# duplicate primary id resolves to the same last-by-position winner the
|
|
110
119
|
# serial crawl would pick, regardless of worker completion order.
|
|
111
120
|
@seen = {}
|
|
121
|
+
# output file => the item #write_file last wrote there, and
|
|
122
|
+
# output file => [[pos, translation item], ...] for #merge_translations.
|
|
123
|
+
@items = {}
|
|
124
|
+
@translations = Hash.new { |h, k| h[k] = [] }
|
|
112
125
|
end
|
|
113
126
|
|
|
114
127
|
# Factory for a fresh stealth browser agent. The pool builds one per worker
|
|
@@ -243,14 +256,15 @@ module Relaton
|
|
|
243
256
|
|
|
244
257
|
# @param doc [Mechanize::Page]
|
|
245
258
|
# @return [Array<Relaton::Bib::Title>]
|
|
246
|
-
def fetch_title(doc)
|
|
259
|
+
def fetch_title(doc, lang = nil)
|
|
247
260
|
t = doc.at_xpath("//hgroup/h2/text()|//hgroup/h1/text()")
|
|
248
261
|
unless t && !t.text.strip.empty?
|
|
249
262
|
@errors[:title] &&= true
|
|
250
263
|
return []
|
|
251
264
|
end
|
|
252
265
|
|
|
253
|
-
|
|
266
|
+
script = script_for(lang) if lang
|
|
267
|
+
result = Bib::Title.from_string t.text.strip, lang, script
|
|
254
268
|
@errors[:title] &&= result.empty?
|
|
255
269
|
result
|
|
256
270
|
end
|
|
@@ -310,20 +324,45 @@ module Relaton
|
|
|
310
324
|
[Bib::Uri.new(type: "src", content: url)]
|
|
311
325
|
end
|
|
312
326
|
|
|
313
|
-
#
|
|
327
|
+
# An XPath test that the element's class attribute has the exact token.
|
|
328
|
+
def self.class_token(token)
|
|
329
|
+
"contains(concat(' ', normalize-space(@class), ' '), ' #{token} ')"
|
|
330
|
+
end
|
|
331
|
+
|
|
332
|
+
# The document description: the div whose class has the exact token
|
|
333
|
+
# `description`. A class substring match also takes the
|
|
334
|
+
# `available-format-description` divs of the "Document Delivery Options"
|
|
335
|
+
# popup, and wrote a format blurb as the abstract (relaton-cie#7).
|
|
336
|
+
ABSTRACT_XPATH = "//div[#{class_token 'description'}]".freeze
|
|
337
|
+
# The text of the description, without its "See more" toggle link.
|
|
338
|
+
ABSTRACT_TEXT_XPATH =
|
|
339
|
+
".//text()[not(ancestor::a[#{class_token 'full-description__more'}]) " \
|
|
340
|
+
"and not(ancestor::script) and not(ancestor::style)]".freeze
|
|
341
|
+
|
|
342
|
+
# @param doc [Moxml::Document, Nokogiri::HTML::Document]
|
|
314
343
|
# @return [Array<Relaton::Bib::LocalizedMarkedUpString>]
|
|
315
|
-
def fetch_abstract(doc)
|
|
316
|
-
content = doc
|
|
344
|
+
def fetch_abstract(doc, lang = "en")
|
|
345
|
+
content = abstract_text doc
|
|
317
346
|
if content.nil? || content.empty?
|
|
318
347
|
@errors[:abstract] &&= true
|
|
319
348
|
return []
|
|
320
349
|
end
|
|
321
350
|
|
|
322
|
-
result = [Bib::Abstract.new(content: content, language:
|
|
351
|
+
result = [Bib::Abstract.new(content: content, language: lang,
|
|
352
|
+
script: script_for(lang))]
|
|
323
353
|
@errors[:abstract] &&= result.empty?
|
|
324
354
|
result
|
|
325
355
|
end
|
|
326
356
|
|
|
357
|
+
# @param doc [Moxml::Document, Nokogiri::HTML::Document]
|
|
358
|
+
# @return [String, nil]
|
|
359
|
+
def abstract_text(doc)
|
|
360
|
+
div = doc.at_xpath ABSTRACT_XPATH
|
|
361
|
+
return unless div
|
|
362
|
+
|
|
363
|
+
div.xpath(ABSTRACT_TEXT_XPATH).map(&:text).join.strip
|
|
364
|
+
end
|
|
365
|
+
|
|
327
366
|
# @param doc [Mechanize::Page]
|
|
328
367
|
# @return [Array<Relaton::Bib::Contributor>]
|
|
329
368
|
def fetch_contributor(doc) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength,Metrics/CyclomaticComplexity
|
|
@@ -391,6 +430,7 @@ module Relaton
|
|
|
391
430
|
# Only the last-by-position hit writes the shared file's content.
|
|
392
431
|
return if superseded? file, pos
|
|
393
432
|
|
|
433
|
+
@items[file] = bib
|
|
394
434
|
if @files.include? file
|
|
395
435
|
Util.warn do
|
|
396
436
|
"File #{file} exists. Docid: #{bib.docidentifier[0].content}\n" \
|
|
@@ -416,6 +456,86 @@ module Relaton
|
|
|
416
456
|
false
|
|
417
457
|
end
|
|
418
458
|
|
|
459
|
+
# The language of a translation hit, or nil for an English one. The
|
|
460
|
+
# listing URL's slug names it (`…-de?product_id=`; English pages end in
|
|
461
|
+
# `-en` or have no slug); a `(DE)` suffix in the code is the fallback.
|
|
462
|
+
#
|
|
463
|
+
# @param hit [Moxml::Element]
|
|
464
|
+
# @return [String, nil]
|
|
465
|
+
def translation_language(hit)
|
|
466
|
+
link = hit.at_xpath(".//h3/a")
|
|
467
|
+
code = link.text.strip.delete_suffix("\u25b9").strip
|
|
468
|
+
lang = link[:href].to_s[/-([a-z]{2})\?product_id=/, 1] ||
|
|
469
|
+
code[/\(([a-z]{2})\)\z/i, 1]&.downcase
|
|
470
|
+
lang if TRANSLATION_LANGUAGES.include?(lang)
|
|
471
|
+
end
|
|
472
|
+
|
|
473
|
+
# @param lang [String, nil]
|
|
474
|
+
# @return [String] the ISO 15924 script of the language
|
|
475
|
+
def script_for(lang) = SCRIPTS.fetch(lang, "Latn")
|
|
476
|
+
|
|
477
|
+
# Merge every deferred translation into the English record of its file
|
|
478
|
+
# (relaton-cie#13), in catalogue order so the output does not depend on
|
|
479
|
+
# worker completion order. A file with no English record keeps its first
|
|
480
|
+
# translation as the record, so no document is lost.
|
|
481
|
+
def merge_translations
|
|
482
|
+
@translations.each { |file, list| merge_file_translations file, list }
|
|
483
|
+
end
|
|
484
|
+
|
|
485
|
+
# One file's merge. A failure is logged and stops only this file, so the
|
|
486
|
+
# crawl still saves its index (#fetch calls index.save next).
|
|
487
|
+
def merge_file_translations(file, list)
|
|
488
|
+
sorted = list.sort_by(&:first).map(&:last)
|
|
489
|
+
base = translation_base file, sorted
|
|
490
|
+
return if sorted.empty?
|
|
491
|
+
|
|
492
|
+
sorted.each { |trans| merge_translation base, trans }
|
|
493
|
+
File.write file, serialize(base), encoding: "UTF-8"
|
|
494
|
+
rescue StandardError => e
|
|
495
|
+
Util.error { "Translations of #{file}: #{e.message}" }
|
|
496
|
+
end
|
|
497
|
+
|
|
498
|
+
# The English record of `file`, or else the first translation in
|
|
499
|
+
# `sorted`, which is then written as the record and taken off the list.
|
|
500
|
+
def translation_base(file, sorted)
|
|
501
|
+
@items[file] || sorted.shift.tap { |item| write_file item }
|
|
502
|
+
end
|
|
503
|
+
|
|
504
|
+
# A translation is its own publication (its own ISBN, date and product
|
|
505
|
+
# page), with no CIE number of its own. So the English record keeps its
|
|
506
|
+
# identity, takes the translated title, and links the translation as a
|
|
507
|
+
# `hasTranslation` relation that keeps the translation's own metadata.
|
|
508
|
+
#
|
|
509
|
+
# @param base [Relaton::Cie::ItemData] the record that stays
|
|
510
|
+
# @param trans [Relaton::Cie::ItemData] its translation
|
|
511
|
+
def merge_translation(base, trans)
|
|
512
|
+
base.title = merge_list(base.title, trans.title,
|
|
513
|
+
:type, :content, :language)
|
|
514
|
+
rel = Bib::Relation.new(type: "hasTranslation",
|
|
515
|
+
bibitem: translation_bibitem(trans))
|
|
516
|
+
base.relation = merge_relation(base.relation, rel)
|
|
517
|
+
end
|
|
518
|
+
|
|
519
|
+
# Add `rel`, unless a relation of its type already links its source.
|
|
520
|
+
def merge_relation(list, rel)
|
|
521
|
+
(list + [rel]).uniq { |r| [r.type, r.bibitem&.source&.map(&:content)] }
|
|
522
|
+
end
|
|
523
|
+
|
|
524
|
+
# The translation as a relation's bibitem. Its abstract is left out: the
|
|
525
|
+
# catalogue shows only a placeholder text on a translation's page.
|
|
526
|
+
def translation_bibitem(trans)
|
|
527
|
+
ItemData.new(
|
|
528
|
+
type: trans.type, docidentifier: trans.docidentifier,
|
|
529
|
+
title: trans.title, date: trans.date, edition: trans.edition,
|
|
530
|
+
source: trans.source, language: trans.language, script: trans.script
|
|
531
|
+
)
|
|
532
|
+
end
|
|
533
|
+
|
|
534
|
+
# Concatenate two element lists, without an element that repeats `keys`.
|
|
535
|
+
def merge_list(list, other, *keys)
|
|
536
|
+
(list + other).uniq { |e| keys.map { |k| e.send(k) } }
|
|
537
|
+
end
|
|
538
|
+
|
|
419
539
|
def to_xml(bib) = bib.to_xml(bibdata: true)
|
|
420
540
|
def to_yaml(bib) = bib.to_yaml
|
|
421
541
|
def to_bibxml(bib) = bib.to_rfcxml
|
|
@@ -431,17 +551,18 @@ module Relaton
|
|
|
431
551
|
# rubocop:disable Metrics/AbcSize, Metrics/MethodLength
|
|
432
552
|
def parse_page(hit, pos = nil, worker_agent = agent, pacing = nil)
|
|
433
553
|
url = hit.at_xpath('.//h3/a')[:href]
|
|
554
|
+
lang = translation_language hit
|
|
434
555
|
doc = time_req(pacing) { worker_agent.get url }
|
|
435
556
|
@mutex.synchronize do
|
|
436
557
|
item = ItemData.new(
|
|
437
558
|
type: "standard", source: fetch_source(url), docnumber: fetch_docnumber(hit),
|
|
438
|
-
docidentifier: fetch_docid(hit, doc), title: fetch_title(doc),
|
|
439
|
-
abstract: fetch_abstract(doc
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
ext: fetch_ext
|
|
559
|
+
docidentifier: fetch_docid(hit, doc), title: fetch_title(doc, lang),
|
|
560
|
+
abstract: fetch_abstract(doc, lang || "en"),
|
|
561
|
+
date: fetch_date(doc), edition: fetch_edition(doc),
|
|
562
|
+
contributor: fetch_contributor(doc), relation: fetch_relation(doc),
|
|
563
|
+
language: lang || "en", script: script_for(lang), ext: fetch_ext
|
|
443
564
|
)
|
|
444
|
-
|
|
565
|
+
lang ? defer_translation(item, pos) : write_file(item, pos)
|
|
445
566
|
end
|
|
446
567
|
rescue StandardError => e
|
|
447
568
|
Util.error do
|
|
@@ -450,8 +571,16 @@ module Relaton
|
|
|
450
571
|
end
|
|
451
572
|
# rubocop:enable Metrics/AbcSize, Metrics/MethodLength
|
|
452
573
|
|
|
574
|
+
# A translation has its English document's primary id: keep it for
|
|
575
|
+
# #merge_translations instead of letting it overwrite that record.
|
|
576
|
+
def defer_translation(item, pos)
|
|
577
|
+
file = output_file item.docidentifier[0].content
|
|
578
|
+
@translations[file] << [pos || 0, item]
|
|
579
|
+
end
|
|
580
|
+
|
|
453
581
|
def fetch(_source = nil)
|
|
454
582
|
process_hits collect_hits
|
|
583
|
+
merge_translations
|
|
455
584
|
index.save
|
|
456
585
|
report_errors
|
|
457
586
|
ensure
|
data/lib/relaton/cli/command.rb
CHANGED
|
@@ -243,9 +243,13 @@ module Relaton
|
|
|
243
243
|
"also works). Default: author-date"
|
|
244
244
|
option :output, aliases: :o, desc: "Output to the specified file"
|
|
245
245
|
|
|
246
|
-
def convert(file) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
|
|
247
|
-
|
|
248
|
-
item
|
|
246
|
+
def convert(file) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
|
|
247
|
+
item = Relaton::Cli.parse_xml File.read(file, encoding: "UTF-8")
|
|
248
|
+
unless item
|
|
249
|
+
Util.warn "#{file} is not a readable Relaton XML document"
|
|
250
|
+
return
|
|
251
|
+
end
|
|
252
|
+
|
|
249
253
|
result = if /yaml|yml/.match?(options[:format])
|
|
250
254
|
item.to_yaml
|
|
251
255
|
elsif options[:format] == "citation"
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
require "
|
|
1
|
+
require "moxml"
|
|
2
2
|
require "pathname"
|
|
3
3
|
|
|
4
4
|
module Relaton
|
|
@@ -89,16 +89,6 @@ module Relaton
|
|
|
89
89
|
)
|
|
90
90
|
end
|
|
91
91
|
|
|
92
|
-
# Turn an XML string into a Nokogiri XML document.
|
|
93
|
-
#
|
|
94
|
-
# @param document [String] XML
|
|
95
|
-
# @param file [String, nil] path to file
|
|
96
|
-
# @return [Nokogiri::XML::Document]
|
|
97
|
-
def nokogiri_document(document, file = nil)
|
|
98
|
-
document ||= File.read(file, encoding: "utf-8")
|
|
99
|
-
Nokogiri.XML(document)
|
|
100
|
-
end
|
|
101
|
-
|
|
102
92
|
def select_source_files
|
|
103
93
|
if File.file?(source)
|
|
104
94
|
[source]
|
|
@@ -108,33 +98,35 @@ module Relaton
|
|
|
108
98
|
end
|
|
109
99
|
|
|
110
100
|
def relaton_collection
|
|
111
|
-
@relaton_collection ||=
|
|
101
|
+
@relaton_collection ||=
|
|
102
|
+
Bibcollection.from_xml(File.read(source, encoding: "utf-8"))
|
|
112
103
|
end
|
|
113
104
|
|
|
114
|
-
def extract_and_write_to_files
|
|
105
|
+
def extract_and_write_to_files
|
|
115
106
|
select_source_files.each do |file|
|
|
116
|
-
xml =
|
|
117
|
-
xml.
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
107
|
+
xml = extract_bibdata_xml(File.read(file, encoding: "utf-8"), file)
|
|
108
|
+
bibdata = xml && Relaton::Bibdata.from_xml(xml)
|
|
109
|
+
next unless bibdata
|
|
110
|
+
|
|
111
|
+
build_bibdata_relaton(bibdata, file)
|
|
112
|
+
write_to_file(bibdata.send(output_type), outdir, build_filename(file))
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
# The bibdata XML of the first `bibdata` element, or of the first `rfc`
|
|
117
|
+
# element, in any namespace.
|
|
118
|
+
#
|
|
119
|
+
# @param content [String] XML
|
|
120
|
+
# @param file [String] path to file
|
|
121
|
+
# @return [String, nil]
|
|
122
|
+
def extract_bibdata_xml(content, file)
|
|
123
|
+
doc = parse_xml_document(content, file)
|
|
124
|
+
return unless doc
|
|
125
|
+
|
|
126
|
+
if (bib = find_element(doc, "bibdata"))
|
|
127
|
+
element_xml(bib)
|
|
128
|
+
elsif (rfc = find_element(doc, "rfc"))
|
|
129
|
+
rfc_bibdata_xml(element_xml(rfc))
|
|
138
130
|
end
|
|
139
131
|
end
|
|
140
132
|
|
|
@@ -142,21 +134,79 @@ require_relative "../ietf/bibxml_parser"
|
|
|
142
134
|
#
|
|
143
135
|
# @return [Array<Relaton::Bibdata>]
|
|
144
136
|
def concatenate_files
|
|
145
|
-
xml_files = [convert_rxl_to_xml, convert_yamls_to_xml,
|
|
137
|
+
xml_files = [convert_rxl_to_xml, convert_yamls_to_xml,
|
|
138
|
+
convert_xml_to_xml]
|
|
146
139
|
|
|
147
140
|
xml_files.flatten.reduce([]) do |mem, xml|
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
141
|
+
bib = concatenate_bibdata_xml(xml[:content], xml[:file])
|
|
142
|
+
bib ? mem << bibdata_instance(bib, xml[:file]) : mem
|
|
143
|
+
end
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
# The bibdata XML of a document whose root is `rfc`, `bibitem` or
|
|
147
|
+
# `bibdata`, in any namespace.
|
|
148
|
+
#
|
|
149
|
+
# @param content [String] XML
|
|
150
|
+
# @param file [String] path to file
|
|
151
|
+
# @return [String, nil]
|
|
152
|
+
def concatenate_bibdata_xml(content, file)
|
|
153
|
+
root = parse_xml_document(content, file)&.root
|
|
154
|
+
case root&.name
|
|
155
|
+
when "rfc" then rfc_bibdata_xml(element_xml(root))
|
|
156
|
+
when "bibitem", "bibdata" then element_xml(root)
|
|
157
|
+
end
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
# Moxml does not recover a malformed document: warn
|
|
161
|
+
# and skip the file. An empty file is skipped without a warning.
|
|
162
|
+
#
|
|
163
|
+
# @param content [String] XML
|
|
164
|
+
# @param file [String] path to file
|
|
165
|
+
# @return [Moxml::Document, nil] nil when the content is not XML
|
|
166
|
+
def parse_xml_document(content, file)
|
|
167
|
+
Moxml.new.parse(content)
|
|
168
|
+
rescue Moxml::ParseError => e
|
|
169
|
+
unless content.to_s.strip.empty?
|
|
170
|
+
Util.warn "Skipping #{file}: not well-formed XML " \
|
|
171
|
+
"(#{e.message.lines.first&.strip})"
|
|
172
|
+
end
|
|
173
|
+
nil
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
# The root element when it has the name, else the first descendant
|
|
177
|
+
# element with that local name, in any namespace.
|
|
178
|
+
#
|
|
179
|
+
# @param doc [Moxml::Document]
|
|
180
|
+
# @param name [String]
|
|
181
|
+
# @return [Moxml::Element, nil]
|
|
182
|
+
def find_element(doc, name)
|
|
183
|
+
return doc.root if doc.root&.name == name
|
|
184
|
+
|
|
185
|
+
doc.xpath("//*[local-name()='#{name}']").first
|
|
186
|
+
end
|
|
187
|
+
|
|
188
|
+
# Serialize an element as a self-contained document: a sub-element
|
|
189
|
+
# inherits its namespace declarations from an ancestor, and `to_xml`
|
|
190
|
+
# does not write them.
|
|
191
|
+
#
|
|
192
|
+
# @param element [Moxml::Element]
|
|
193
|
+
# @return [String]
|
|
194
|
+
def element_xml(element)
|
|
195
|
+
declared = element.namespace_definitions.map(&:prefix)
|
|
196
|
+
element.in_scope_namespaces.each do |ns|
|
|
197
|
+
next if declared.include?(ns.prefix)
|
|
198
|
+
|
|
199
|
+
element.add_namespace(ns.prefix, ns.uri)
|
|
159
200
|
end
|
|
201
|
+
element.to_xml
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
# @param rfc [String] RFC XML
|
|
205
|
+
# @return [String] bibdata XML
|
|
206
|
+
def rfc_bibdata_xml(rfc)
|
|
207
|
+
require_relative "../ietf"
|
|
208
|
+
require_relative "../ietf/bibxml_parser"
|
|
209
|
+
Relaton::Ietf::BibXMLParser.parse_rfc(rfc).to_xml(bibdata: true)
|
|
160
210
|
end
|
|
161
211
|
|
|
162
212
|
def concatenate_and_write_to_files
|
|
@@ -194,16 +244,15 @@ require_relative "../ietf/bibxml_parser"
|
|
|
194
244
|
end
|
|
195
245
|
end
|
|
196
246
|
|
|
197
|
-
# Create a bibdata instance from the XML
|
|
247
|
+
# Create a bibdata instance from the XML,
|
|
198
248
|
# while also adding file paths of related artifacts
|
|
199
249
|
# as URI elements to the bibdata instance.
|
|
200
250
|
#
|
|
201
|
-
# @param
|
|
251
|
+
# @param xml [String] bibdata XML
|
|
202
252
|
# @param file [String] path to file
|
|
203
253
|
# @return [Relaton::Bibdata]
|
|
204
|
-
def bibdata_instance(
|
|
205
|
-
|
|
206
|
-
bibdata = Relaton::Bibdata.from_xml document.root
|
|
254
|
+
def bibdata_instance(xml, file)
|
|
255
|
+
bibdata = Relaton::Bibdata.from_xml xml
|
|
207
256
|
build_bibdata_relaton(bibdata, file) if bibdata
|
|
208
257
|
|
|
209
258
|
bibdata
|
|
@@ -223,18 +272,6 @@ require_relative "../ietf/bibxml_parser"
|
|
|
223
272
|
end
|
|
224
273
|
end
|
|
225
274
|
|
|
226
|
-
# Force a namespace otherwise Nokogiri won't parse.
|
|
227
|
-
# The reason is we use Bibcollection's from_xml, but that one
|
|
228
|
-
# has an xmlns. We don't want to change the code for bibdata
|
|
229
|
-
# hence this hack #bibdata_doc.root['xmlns'] = "xmlns"
|
|
230
|
-
#
|
|
231
|
-
# @param document [Nokogiri::XML::Document]
|
|
232
|
-
# @return [Nokogiri::XML::Document]
|
|
233
|
-
def clean_nokogiri_document(document)
|
|
234
|
-
document.remove_namespaces!
|
|
235
|
-
nokogiri_document(document.to_xml)
|
|
236
|
-
end
|
|
237
|
-
|
|
238
275
|
def convert_rxl_to_xml
|
|
239
276
|
select_files_with("{rxl}").map do |file|
|
|
240
277
|
{ file: file, content: File.read(file, encoding: "utf-8") }
|
|
@@ -146,8 +146,8 @@ module Relaton
|
|
|
146
146
|
|
|
147
147
|
def import(file)
|
|
148
148
|
xml = read_xml file
|
|
149
|
-
if xml.
|
|
150
|
-
imported =
|
|
149
|
+
if Moxml.new.parse(xml).root&.name == "relaton-collection"
|
|
150
|
+
imported = Relaton::Bibcollection.from_xml(xml)
|
|
151
151
|
store.update(options[:collection]) do |coll|
|
|
152
152
|
next imported unless coll # a new collection is the imported one
|
|
153
153
|
|
|
@@ -173,20 +173,10 @@ module Relaton
|
|
|
173
173
|
|
|
174
174
|
private
|
|
175
175
|
|
|
176
|
-
#
|
|
177
|
-
#
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
title = xml.at("relaton-collection/title")&.text
|
|
181
|
-
author = xml.at_xpath(
|
|
182
|
-
"./relaton-collection/contributor[role/@type='author']"\
|
|
183
|
-
"/organization/name",
|
|
184
|
-
)&.text
|
|
185
|
-
items = xml.xpath("relaton-collection/relation").map do |rel|
|
|
186
|
-
el = rel.at("bibdata") || rel.at("bibitem")
|
|
187
|
-
Relaton::Bibdata.from_xml(el || rel)
|
|
188
|
-
end.compact
|
|
189
|
-
Relaton::Bibcollection.new(title: title, author: author, items: items)
|
|
176
|
+
# The XML file `import` reads: user input, not a collection.
|
|
177
|
+
# @return [String]
|
|
178
|
+
def read_xml(file)
|
|
179
|
+
File.read file, encoding: "UTF-8"
|
|
190
180
|
end
|
|
191
181
|
|
|
192
182
|
# @return [String]
|
|
@@ -210,14 +200,6 @@ module Relaton
|
|
|
210
200
|
end
|
|
211
201
|
end
|
|
212
202
|
|
|
213
|
-
# The XML file `import` reads: user input, not a collection.
|
|
214
|
-
# @return [Nokogiri::XML::Document]
|
|
215
|
-
def read_xml(file)
|
|
216
|
-
xml = Nokogiri::XML File.read(file, encoding: "UTF-8")
|
|
217
|
-
xml.remove_namespaces!
|
|
218
|
-
xml
|
|
219
|
-
end
|
|
220
|
-
|
|
221
203
|
# @return [Array<Hash>]
|
|
222
204
|
def collections
|
|
223
205
|
store.names(options.fetch(:collection, "*")).filter_map do |name|
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
require "
|
|
1
|
+
require "moxml"
|
|
2
2
|
require_relative "base_convertor"
|
|
3
3
|
|
|
4
4
|
module Relaton
|
|
@@ -29,10 +29,11 @@ module Relaton
|
|
|
29
29
|
"yaml"
|
|
30
30
|
end
|
|
31
31
|
|
|
32
|
-
# @param content [
|
|
33
|
-
#
|
|
32
|
+
# @param content [String] Relaton XML: a bibliographic item or a
|
|
33
|
+
# collection
|
|
34
|
+
# @return [Relaton::Bibdata, Relaton::Bibcollection, nil]
|
|
34
35
|
def convert_content(content)
|
|
35
|
-
if %w[bibitem bibdata].include? content.root
|
|
36
|
+
if %w[bibitem bibdata].include? Moxml.new.parse(content).root&.name
|
|
36
37
|
Bibdata.from_xml(content)
|
|
37
38
|
else
|
|
38
39
|
Bibcollection.from_xml(content)
|
|
@@ -40,7 +41,7 @@ module Relaton
|
|
|
40
41
|
end
|
|
41
42
|
|
|
42
43
|
def file_content
|
|
43
|
-
|
|
44
|
+
File.read(file, encoding: "utf-8")
|
|
44
45
|
end
|
|
45
46
|
end
|
|
46
47
|
end
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
require "nokogiri"
|
|
2
1
|
require "liquid"
|
|
3
2
|
|
|
4
3
|
module Relaton::Cli
|
|
@@ -112,7 +111,7 @@ module Relaton::Cli
|
|
|
112
111
|
# @param source [String] Relaton XML
|
|
113
112
|
# @return [Relaton::Bibcollection]
|
|
114
113
|
def build_bibcollection(source)
|
|
115
|
-
Relaton::Bibcollection.from_xml(
|
|
114
|
+
Relaton::Bibcollection.from_xml(source)
|
|
116
115
|
end
|
|
117
116
|
|
|
118
117
|
# @param bibcollection [Relaton::Bibcollection]
|