relaton 3.0.0.pre.alpha.4 → 3.0.0.pre.alpha.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/relaton/3gpp/bibliography.rb +19 -6
- data/lib/relaton/3gpp/processor.rb +6 -0
- data/lib/relaton/adobe/processor.rb +9 -0
- data/lib/relaton/bib/converter/csl.rb +110 -0
- data/lib/relaton/bib/converter/ris.rb +104 -0
- data/lib/relaton/bib/converter/titles.rb +24 -0
- data/lib/relaton/bib/item_data.rb +23 -0
- data/lib/relaton/bib/model/item.rb +6 -6
- data/lib/relaton/bib/sanitizer.rb +71 -30
- data/lib/relaton/bib.rb +10 -0
- data/lib/relaton/bipm/bibliography.rb +24 -21
- data/lib/relaton/bipm/processor.rb +1 -0
- data/lib/relaton/bipm/rawdata_bipm_metrologia/affiliations.rb +6 -6
- data/lib/relaton/bipm/si_brochure_parser.rb +2 -2
- data/lib/relaton/bsi/bibliography.rb +26 -23
- data/lib/relaton/bsi/processor.rb +5 -0
- data/lib/relaton/calconnect/bibliography.rb +6 -6
- data/lib/relaton/calconnect/hit_collection.rb +4 -1
- data/lib/relaton/ccsds/bibliography.rb +16 -10
- data/lib/relaton/ccsds/hit_collection.rb +1 -1
- data/lib/relaton/ccsds/processor.rb +13 -0
- data/lib/relaton/cen/bibliography.rb +27 -24
- data/lib/relaton/cen/hit_collection.rb +1 -1
- data/lib/relaton/cen/processor.rb +5 -0
- data/lib/relaton/cen/scraper.rb +9 -9
- data/lib/relaton/cie/bibliography.rb +11 -9
- data/lib/relaton/cie/data_fetcher.rb +18 -18
- data/lib/relaton/cie/processor.rb +1 -0
- data/lib/relaton/cie/scrapper.rb +15 -8
- data/lib/relaton/cie.rb +1 -1
- data/lib/relaton/cloud.rb +127 -0
- data/lib/relaton/core/hit_collection.rb +6 -10
- data/lib/relaton/core/processor.rb +121 -0
- data/lib/relaton/core/request_error.rb +21 -3
- data/lib/relaton/db/cache.rb +474 -148
- data/lib/relaton/db/cache_entry.rb +42 -0
- data/lib/relaton/db/registry.rb +152 -0
- data/lib/relaton/db.rb +275 -102
- data/lib/relaton/doi/crossref.rb +23 -6
- data/lib/relaton/doi/processor.rb +1 -0
- data/lib/relaton/easc/processor.rb +1 -0
- data/lib/relaton/ecma/bibliography.rb +18 -13
- data/lib/relaton/ecma/data_fetcher.rb +1 -1
- data/lib/relaton/ecma/data_parser.rb +1 -1
- data/lib/relaton/ecma/edition_parser.rb +2 -2
- data/lib/relaton/ecma/memento_parser.rb +4 -4
- data/lib/relaton/ecma/standard_parser.rb +6 -6
- data/lib/relaton/etsi/bibliography.rb +8 -7
- data/lib/relaton/etsi/processor.rb +1 -0
- data/lib/relaton/gb/bibliography.rb +20 -12
- data/lib/relaton/gb/gb_scraper.rb +5 -5
- data/lib/relaton/gb/scraper.rb +16 -16
- data/lib/relaton/gb/sec_scraper.rb +8 -8
- data/lib/relaton/gb/t_scraper.rb +5 -5
- data/lib/relaton/gost/processor.rb +1 -0
- data/lib/relaton/iala/bibliography.rb +15 -12
- data/lib/relaton/iala/processor.rb +1 -0
- data/lib/relaton/iana/bibliography.rb +12 -12
- data/lib/relaton/iana/data_fetcher.rb +3 -3
- data/lib/relaton/iana/parser.rb +12 -5
- data/lib/relaton/iana/processor.rb +9 -0
- data/lib/relaton/iec/bibliography.rb +18 -7
- data/lib/relaton/iec/data_parser.rb +24 -9
- data/lib/relaton/iec/processor.rb +11 -0
- data/lib/relaton/iec.rb +1 -1
- data/lib/relaton/ieee/bibliography.rb +19 -13
- data/lib/relaton/ieee/data_fetcher.rb +1 -1
- data/lib/relaton/ieee/processor.rb +24 -0
- data/lib/relaton/ieee/rawbib_id_parser.rb +2 -2
- data/lib/relaton/ietf/bibliography.rb +11 -9
- data/lib/relaton/ietf/data_fetcher.rb +1 -1
- data/lib/relaton/ietf/processor.rb +1 -0
- data/lib/relaton/ietf/rfc/entry.rb +15 -19
- data/lib/relaton/ietf/scraper.rb +5 -4
- data/lib/relaton/iho/processor.rb +1 -0
- data/lib/relaton/index/file_io.rb +2 -2
- data/lib/relaton/index/pool.rb +4 -3
- data/lib/relaton/index/shard_source.rb +1 -1
- data/lib/relaton/index/type.rb +2 -5
- data/lib/relaton/isbn/open_library.rb +11 -7
- data/lib/relaton/isbn/processor.rb +10 -0
- data/lib/relaton/iso/bibliography.rb +11 -9
- data/lib/relaton/iso/data_parser.rb +2 -2
- data/lib/relaton/iso/processor.rb +5 -0
- data/lib/relaton/iso/scraper.rb +20 -20
- data/lib/relaton/itu/bibliography.rb +107 -16
- data/lib/relaton/itu/data_crawler_r.rb +2 -2
- data/lib/relaton/itu/hit_collection.rb +64 -52
- data/lib/relaton/itu/processor.rb +1 -0
- data/lib/relaton/itu/scraper.rb +13 -3
- data/lib/relaton/itu.rb +0 -2
- data/lib/relaton/jis/bibliography.rb +22 -11
- data/lib/relaton/jis/data_fetcher.rb +4 -4
- data/lib/relaton/jis/processor.rb +1 -0
- data/lib/relaton/jis/scraper.rb +8 -8
- data/lib/relaton/nist/bibliography.rb +25 -21
- data/lib/relaton/oasis/bibliography.rb +16 -13
- data/lib/relaton/oasis/browser_agent.rb +2 -2
- data/lib/relaton/oasis/data_parser.rb +5 -5
- data/lib/relaton/oasis/data_parser_utils.rb +2 -2
- data/lib/relaton/oasis/data_part_parser.rb +7 -7
- data/lib/relaton/ogc/bibliography.rb +15 -14
- data/lib/relaton/ogc/hit_collection.rb +9 -7
- data/lib/relaton/ogc/processor.rb +13 -0
- data/lib/relaton/oiml/processor.rb +1 -0
- data/lib/relaton/omg/bibliography.rb +9 -9
- data/lib/relaton/omg/scraper.rb +14 -13
- data/lib/relaton/omg.rb +1 -1
- data/lib/relaton/plateau/bibliography.rb +9 -7
- data/lib/relaton/plateau/hit_collection.rb +2 -1
- data/lib/relaton/plateau/processor.rb +1 -0
- data/lib/relaton/un/bibliography.rb +21 -10
- data/lib/relaton/un/processor.rb +1 -0
- data/lib/relaton/version.rb +1 -1
- data/lib/relaton/w3c/processor.rb +1 -0
- data/lib/relaton/xsf/bibliography.rb +11 -6
- data/lib/relaton.rb +13 -0
- metadata +60 -14
- data/lib/relaton/itu/pubid.rb +0 -199
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
require "lutaml/model"
|
|
2
|
+
|
|
3
|
+
module Relaton
|
|
4
|
+
class Db
|
|
5
|
+
# The cached answer "the flavor has no such document", with the date it
|
|
6
|
+
# was fetched. Db and Cache pass it in place of the document XML.
|
|
7
|
+
NotFound = Data.define(:fetched) do
|
|
8
|
+
# @return [String] the legacy string form, `not_found <date>`
|
|
9
|
+
def to_s
|
|
10
|
+
"not_found #{fetched}"
|
|
11
|
+
end
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
# One row of the cache index. A row is keyed either by a pubid (`id`, its
|
|
15
|
+
# `to_hash`) or, for a processor with no pubid class, by a string (`key`).
|
|
16
|
+
# `file` names the document in the doc store; several rows can share one
|
|
17
|
+
# file (a query row and the row of the document it returned). A
|
|
18
|
+
# `not_found` row has no file.
|
|
19
|
+
class CacheEntry < Lutaml::Model::Serializable
|
|
20
|
+
DOC = "doc".freeze
|
|
21
|
+
NOT_FOUND = "not_found".freeze
|
|
22
|
+
|
|
23
|
+
attribute :id, :hash
|
|
24
|
+
attribute :key, :string
|
|
25
|
+
attribute :status, :string
|
|
26
|
+
attribute :file, :string
|
|
27
|
+
attribute :fetched, :string
|
|
28
|
+
|
|
29
|
+
key_value do
|
|
30
|
+
map "id", to: :id
|
|
31
|
+
map "key", to: :key
|
|
32
|
+
map "status", to: :status
|
|
33
|
+
map "file", to: :file
|
|
34
|
+
map "fetched", to: :fetched
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def not_found?
|
|
38
|
+
status == NOT_FOUND
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
end
|
data/lib/relaton/db/registry.rb
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
require "singleton"
|
|
2
|
+
require_relative "../core/request_error"
|
|
2
3
|
|
|
3
4
|
module Relaton
|
|
4
5
|
class Db
|
|
@@ -92,6 +93,63 @@ module Relaton
|
|
|
92
93
|
end
|
|
93
94
|
|
|
94
95
|
#
|
|
96
|
+
# The processor whose pubid class the identifier belongs to. Only the
|
|
97
|
+
# generic `Pubid::AllPartsIdentifier`, which belongs to no flavor, is
|
|
98
|
+
# matched through the document it wraps (`#root`). Any other identifier
|
|
99
|
+
# is matched by its own class: the `#root` of an adoption is the adopted
|
|
100
|
+
# document, so `CEN ISO/TS 21003-7` would otherwise be filed as ISO.
|
|
101
|
+
#
|
|
102
|
+
# @param pubid [Pubid::Identifier]
|
|
103
|
+
# @return [Relaton::Core::Processor, nil]
|
|
104
|
+
#
|
|
105
|
+
def processor_by_pubid(pubid)
|
|
106
|
+
generic = pubid.instance_of?(::Pubid::AllPartsIdentifier)
|
|
107
|
+
id = generic ? pubid.root : pubid
|
|
108
|
+
processors.values.detect do |processor|
|
|
109
|
+
klass = processor.pubid_class
|
|
110
|
+
klass && id.is_a?(klass)
|
|
111
|
+
end
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
#
|
|
115
|
+
# Route a reference to its flavor (relaton#205).
|
|
116
|
+
#
|
|
117
|
+
# 1. The reference is parsed with pubid. Only the parsing flavor's own
|
|
118
|
+
# **exact** parse counts (#exact_parse): `Pubid.parse` falls back to a
|
|
119
|
+
# partial parse by any flavor (`ATN5014` as `IEC ATN5014`), and a
|
|
120
|
+
# permissive grammar reads another publisher's string exactly
|
|
121
|
+
# (`ISO REF` as IEC).
|
|
122
|
+
# 2. The parse routes by class ancestry (#processor_by_pubid), so the
|
|
123
|
+
# registration order does not matter.
|
|
124
|
+
# 3. A co-published identifier routes to the co-publisher its printed
|
|
125
|
+
# form names first (`ISO/IEC …` → ISO, `IEC/ISO …` → IEC). This is
|
|
126
|
+
# relaton's routing policy (#205): there is no canonical form of the
|
|
127
|
+
# identifier, pubid does not route records (pubid#469), and
|
|
128
|
+
# `Pubid.parse` tries the owners of a joint prefix in alphabetical
|
|
129
|
+
# order.
|
|
130
|
+
# 4. Otherwise the prefix regex (#class_by_ref) routes: a combined
|
|
131
|
+
# reference, the `PREFIX(code)` wrapper, a spelling a flavor
|
|
132
|
+
# normalizes, a flavor's free text.
|
|
133
|
+
#
|
|
134
|
+
# @param reference [String]
|
|
135
|
+
# @return [Array(Symbol, Pubid::Identifier)] the standard class, and the
|
|
136
|
+
# parsed pubid when it is that flavor's own exact parse (else nil)
|
|
137
|
+
# @raise [Relaton::UnknownReferenceError] no flavor recognizes it
|
|
138
|
+
#
|
|
139
|
+
def route(reference) # rubocop:disable Metrics/CyclomaticComplexity
|
|
140
|
+
parsed = exact_parse reference
|
|
141
|
+
stdclass = parsed && class_by_parsed(parsed, reference)
|
|
142
|
+
parsed = nil unless stdclass
|
|
143
|
+
lead = joint_lead reference
|
|
144
|
+
return [lead, stdclass == lead ? parsed : nil] if lead
|
|
145
|
+
return [stdclass, parsed] if stdclass
|
|
146
|
+
|
|
147
|
+
stdclass = class_by_ref(reference)
|
|
148
|
+
raise UnknownReferenceError, reference unless stdclass
|
|
149
|
+
|
|
150
|
+
[stdclass, nil]
|
|
151
|
+
end
|
|
152
|
+
|
|
95
153
|
# Find processor by refernce or prefix
|
|
96
154
|
#
|
|
97
155
|
# @param ref [String] reference or prefix
|
|
@@ -137,8 +195,102 @@ module Relaton
|
|
|
137
195
|
processors_by_prefix(prefix).map { |processor| flavor_module(processor) }
|
|
138
196
|
end
|
|
139
197
|
|
|
198
|
+
#
|
|
199
|
+
# The flavor that serves a publisher's catalog: the processor whose own
|
|
200
|
+
# prefix is the publisher (`IEC` → `:relaton_iec`). Nil for a publisher
|
|
201
|
+
# no flavor serves (`ASTM`). Used for a co-publisher (relaton#205).
|
|
202
|
+
#
|
|
203
|
+
# @param publisher [String]
|
|
204
|
+
# @return [Symbol, nil] the standard class
|
|
205
|
+
#
|
|
206
|
+
def class_by_publisher(publisher)
|
|
207
|
+
processors_by_prefix(publisher)
|
|
208
|
+
.detect { |processor| processor.prefix.casecmp?(publisher) }&.short
|
|
209
|
+
end
|
|
210
|
+
|
|
140
211
|
private
|
|
141
212
|
|
|
213
|
+
# The reference parsed by pubid, when the parse renders the reference
|
|
214
|
+
# back (pubid's round-trip test). pubid itself is loaded here, not at
|
|
215
|
+
# registration. `Pubid.parse` raises a plain ArgumentError for a URN no
|
|
216
|
+
# flavor owns (`urn:foo:bar`): that is no parse either.
|
|
217
|
+
#
|
|
218
|
+
# @param reference [String]
|
|
219
|
+
# @return [Pubid::Identifier, nil]
|
|
220
|
+
def exact_parse(reference)
|
|
221
|
+
require "pubid"
|
|
222
|
+
parsed = ::Pubid.parse reference
|
|
223
|
+
parsed if parsed.to_s == reference
|
|
224
|
+
rescue ArgumentError, ::Pubid::Errors::Error, Parslet::ParseFailed
|
|
225
|
+
nil
|
|
226
|
+
end
|
|
227
|
+
|
|
228
|
+
# Whether the flavor of +pubid+ owns a prefix +reference+ starts with.
|
|
229
|
+
def claims?(pubid, reference)
|
|
230
|
+
flavor = ::Pubid.const_get pubid.class.name.split("::")[1]
|
|
231
|
+
return false unless flavor.respond_to?(:prefixes)
|
|
232
|
+
|
|
233
|
+
flavor.prefixes.any? { |pref| prefix_of?(pref, reference) }
|
|
234
|
+
end
|
|
235
|
+
|
|
236
|
+
# A prefix matches when the reference ends there or goes on with a
|
|
237
|
+
# separator, or when the prefix itself ends with one (`doi:10.…`).
|
|
238
|
+
def prefix_of?(prefix, reference)
|
|
239
|
+
return false unless reference.start_with?(prefix)
|
|
240
|
+
|
|
241
|
+
rest = reference[prefix.length]
|
|
242
|
+
rest.nil? || !alnum?(rest) || !alnum?(prefix[-1])
|
|
243
|
+
end
|
|
244
|
+
|
|
245
|
+
def alnum?(char)
|
|
246
|
+
char.match?(/[[:alnum:]]/)
|
|
247
|
+
end
|
|
248
|
+
|
|
249
|
+
# The flavor of an exact parse, when the parse is that flavor's own: the
|
|
250
|
+
# flavor claims the reference by one of its prefixes, or its identifiers
|
|
251
|
+
# print without one (Core::Processor#bare_identifiers?). A permissive
|
|
252
|
+
# grammar reads other publishers' strings exactly (`ISO REF` as IEC,
|
|
253
|
+
# `ABC 123456` as GB, a bare DOI as UN), but it does not claim them.
|
|
254
|
+
#
|
|
255
|
+
# @param pubid [Pubid::Identifier]
|
|
256
|
+
# @param reference [String]
|
|
257
|
+
# @return [Symbol, nil] the standard class
|
|
258
|
+
def class_by_parsed(pubid, reference)
|
|
259
|
+
processor = processor_by_pubid(pubid) or return
|
|
260
|
+
return unless processor.bare_identifiers? || claims?(pubid, reference)
|
|
261
|
+
|
|
262
|
+
processor.short
|
|
263
|
+
end
|
|
264
|
+
|
|
265
|
+
# The co-publisher that a joint prefix names first (`ISO/IEC 27001` →
|
|
266
|
+
# ISO). Nil unless the reference starts with a prefix several flavors
|
|
267
|
+
# own.
|
|
268
|
+
#
|
|
269
|
+
# @param reference [String]
|
|
270
|
+
# @return [Symbol, nil] the standard class
|
|
271
|
+
def joint_lead(reference)
|
|
272
|
+
prefix = joint_prefixes.detect do |pref|
|
|
273
|
+
reference == pref || reference.start_with?("#{pref} ")
|
|
274
|
+
end
|
|
275
|
+
return unless prefix
|
|
276
|
+
|
|
277
|
+
class_by_publisher prefix.split("/").first
|
|
278
|
+
end
|
|
279
|
+
|
|
280
|
+
# Prefixes that more than one flavor owns (`ISO/IEC`, `ISO/IEC/IEEE`),
|
|
281
|
+
# longest first.
|
|
282
|
+
#
|
|
283
|
+
# @return [Array<String>]
|
|
284
|
+
def joint_prefixes
|
|
285
|
+
@joint_prefixes ||= processors.values.flat_map(&:prefixes).uniq
|
|
286
|
+
.select { |pref| pref.include?("/") && joint?(pref) }
|
|
287
|
+
.sort_by { |pref| -pref.size }
|
|
288
|
+
end
|
|
289
|
+
|
|
290
|
+
def joint?(prefix)
|
|
291
|
+
processors_by_prefix(prefix).size > 1
|
|
292
|
+
end
|
|
293
|
+
|
|
142
294
|
# The flavor namespace for a processor: the module enclosing its class.
|
|
143
295
|
# Relaton::Iso::Processor -> Relaton::Iso. Derived from the class name so
|
|
144
296
|
# there's no @short string-munging and no 3gpp special case.
|