relaton 2.2.0.pre.alpha.1 → 3.0.0.pre.alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/{docs/README.adoc → README.adoc} +165 -38
- data/bin/console +0 -1
- data/lib/relaton/3gpp/bibdata.rb +9 -0
- data/lib/relaton/3gpp/bibitem.rb +9 -0
- data/lib/relaton/3gpp/bibliography.rb +123 -0
- data/lib/relaton/3gpp/data_fetcher.rb +303 -0
- data/lib/relaton/3gpp/docidentifier.rb +114 -0
- data/lib/relaton/3gpp/doctype.rb +9 -0
- data/lib/relaton/3gpp/ext.rb +31 -0
- data/lib/relaton/3gpp/item.rb +18 -0
- data/lib/relaton/3gpp/item_data.rb +15 -0
- data/lib/relaton/3gpp/parser.rb +400 -0
- data/lib/relaton/3gpp/processor.rb +71 -0
- data/lib/relaton/3gpp/release.rb +34 -0
- data/lib/relaton/3gpp/util.rb +8 -0
- data/lib/relaton/3gpp.rb +29 -0
- data/lib/relaton/adobe/bibdata.rb +8 -0
- data/lib/relaton/adobe/bibitem.rb +8 -0
- data/lib/relaton/adobe/bibliography.rb +92 -0
- data/lib/relaton/adobe/docidentifier.rb +49 -0
- data/lib/relaton/adobe/doctype.rb +14 -0
- data/lib/relaton/adobe/ext.rb +32 -0
- data/lib/relaton/adobe/item.rb +15 -0
- data/lib/relaton/adobe/item_base.rb +18 -0
- data/lib/relaton/adobe/item_data.rb +6 -0
- data/lib/relaton/adobe/processor.rb +45 -0
- data/lib/relaton/adobe/util.rb +8 -0
- data/lib/relaton/adobe.rb +37 -0
- data/lib/relaton/bib/converter/asciibib/to_asciibib.rb +663 -0
- data/lib/relaton/bib/converter/asciibib.rb +13 -0
- data/lib/relaton/bib/converter/bibtex/from_bibtex.rb +245 -0
- data/lib/relaton/bib/converter/bibtex/to_bibtex.rb +341 -0
- data/lib/relaton/bib/converter/bibtex.rb +23 -0
- data/lib/relaton/bib/converter/bibxml/from_rfcxml.rb +386 -0
- data/lib/relaton/bib/converter/bibxml/from_rfcxml_referencegroup.rb +71 -0
- data/lib/relaton/bib/converter/bibxml/to_rfcxml.rb +308 -0
- data/lib/relaton/bib/converter/bibxml/to_rfcxml_referencegroup.rb +52 -0
- data/lib/relaton/bib/converter/bibxml.rb +51 -0
- data/lib/relaton/bib/hash_parser_v1.rb +767 -0
- data/lib/relaton/bib/item_data.rb +229 -0
- data/lib/relaton/bib/model/abstract.rb +16 -0
- data/lib/relaton/bib/model/address.rb +22 -0
- data/lib/relaton/bib/model/affiliation.rb +16 -0
- data/lib/relaton/bib/model/bibdata.rb +11 -0
- data/lib/relaton/bib/model/bibdata_shared.rb +12 -0
- data/lib/relaton/bib/model/bibitem.rb +11 -0
- data/lib/relaton/bib/model/bibitem_shared.rb +12 -0
- data/lib/relaton/bib/model/contact.rb +18 -0
- data/lib/relaton/bib/model/contribution_info.rb +15 -0
- data/lib/relaton/bib/model/contributor.rb +29 -0
- data/lib/relaton/bib/model/copyright.rb +27 -0
- data/lib/relaton/bib/model/date.rb +31 -0
- data/lib/relaton/bib/model/depiction.rb +16 -0
- data/lib/relaton/bib/model/docidentifier.rb +49 -0
- data/lib/relaton/bib/model/doctype.rb +14 -0
- data/lib/relaton/bib/model/edition.rb +14 -0
- data/lib/relaton/bib/model/ext.rb +39 -0
- data/lib/relaton/bib/model/extent.rb +16 -0
- data/lib/relaton/bib/model/formattedref.rb +43 -0
- data/lib/relaton/bib/model/full_name_type.rb +64 -0
- data/lib/relaton/bib/model/fullname.rb +11 -0
- data/lib/relaton/bib/model/ics.rb +49 -0
- data/lib/relaton/bib/model/image.rb +28 -0
- data/lib/relaton/bib/model/item.rb +96 -0
- data/lib/relaton/bib/model/item_base.rb +20 -0
- data/lib/relaton/bib/model/item_shared.rb +88 -0
- data/lib/relaton/bib/model/keyword.rb +30 -0
- data/lib/relaton/bib/model/locality.rb +18 -0
- data/lib/relaton/bib/model/locality_stack.rb +14 -0
- data/lib/relaton/bib/model/localized_string.rb +48 -0
- data/lib/relaton/bib/model/localized_string_attrs.rb +24 -0
- data/lib/relaton/bib/model/logo.rb +14 -0
- data/lib/relaton/bib/model/medium.rb +22 -0
- data/lib/relaton/bib/model/note.rb +16 -0
- data/lib/relaton/bib/model/organization.rb +13 -0
- data/lib/relaton/bib/model/organization_type.rb +42 -0
- data/lib/relaton/bib/model/person.rb +36 -0
- data/lib/relaton/bib/model/phone.rb +14 -0
- data/lib/relaton/bib/model/place.rb +33 -0
- data/lib/relaton/bib/model/price.rb +14 -0
- data/lib/relaton/bib/model/relation.rb +43 -0
- data/lib/relaton/bib/model/series.rb +34 -0
- data/lib/relaton/bib/model/size.rb +23 -0
- data/lib/relaton/bib/model/source_locality_stack.rb +14 -0
- data/lib/relaton/bib/model/status.rb +27 -0
- data/lib/relaton/bib/model/structured_identifier.rb +49 -0
- data/lib/relaton/bib/model/subdivision.rb +16 -0
- data/lib/relaton/bib/model/title.rb +56 -0
- data/lib/relaton/bib/model/type/plain_date.rb +16 -0
- data/lib/relaton/bib/model/type/string_date.rb +48 -0
- data/lib/relaton/bib/model/uri.rb +18 -0
- data/lib/relaton/bib/model/validity.rb +16 -0
- data/lib/relaton/bib/model/version.rb +43 -0
- data/lib/relaton/bib/namespace_helper.rb +21 -0
- data/lib/relaton/bib/sanitizer.rb +264 -0
- data/lib/relaton/bib/util.rb +18 -0
- data/lib/relaton/bib/versions.json +35 -0
- data/lib/relaton/bib.rb +46 -0
- data/lib/relaton/bipm/bibliography.rb +231 -0
- data/lib/relaton/bipm/converter/asciibib.rb +64 -0
- data/lib/relaton/bipm/data_fetcher.rb +81 -0
- data/lib/relaton/bipm/data_outcomes_parser.rb +656 -0
- data/lib/relaton/bipm/id_parser.rb +278 -0
- data/lib/relaton/bipm/item_data.rb +47 -0
- data/lib/relaton/bipm/model/bibdata.rb +9 -0
- data/lib/relaton/bipm/model/bibitem.rb +9 -0
- data/lib/relaton/bipm/model/comment_period.rb +13 -0
- data/lib/relaton/bipm/model/doctype.rb +12 -0
- data/lib/relaton/bipm/model/ext.rb +38 -0
- data/lib/relaton/bipm/model/item.rb +11 -0
- data/lib/relaton/bipm/model/structured_identifier.rb +36 -0
- data/lib/relaton/bipm/processor.rb +69 -0
- data/lib/relaton/bipm/rawdata_bipm_metrologia/affiliations.rb +111 -0
- data/lib/relaton/bipm/rawdata_bipm_metrologia/fetcher.rb +172 -0
- data/lib/relaton/bipm/rawdata_bipm_metrologia/niso_jats_parser.rb +353 -0
- data/lib/relaton/bipm/si_brochure_parser.rb +188 -0
- data/lib/relaton/bipm/util.rb +8 -0
- data/lib/relaton/bipm.rb +38 -0
- data/lib/relaton/bsi/bibliography.rb +196 -0
- data/lib/relaton/bsi/hit.rb +28 -0
- data/lib/relaton/bsi/hit_collection.rb +113 -0
- data/lib/relaton/bsi/item_data.rb +13 -0
- data/lib/relaton/bsi/model/bibdata.rb +8 -0
- data/lib/relaton/bsi/model/bibitem.rb +8 -0
- data/lib/relaton/bsi/model/docidentifier.rb +107 -0
- data/lib/relaton/bsi/model/doctype.rb +14 -0
- data/lib/relaton/bsi/model/ext.rb +17 -0
- data/lib/relaton/bsi/model/item.rb +17 -0
- data/lib/relaton/bsi/model/item_base.rb +22 -0
- data/lib/relaton/bsi/model/relation.rb +9 -0
- data/lib/relaton/bsi/processor.rb +43 -0
- data/lib/relaton/bsi/schema.json +24882 -0
- data/lib/relaton/bsi/scraper.rb +288 -0
- data/lib/relaton/bsi/util.rb +8 -0
- data/lib/relaton/bsi.rb +25 -0
- data/lib/relaton/calconnect/bibliography.rb +93 -0
- data/lib/relaton/calconnect/data_fetcher.rb +174 -0
- data/lib/relaton/calconnect/docidentifier.rb +80 -0
- data/lib/relaton/calconnect/hit.rb +14 -0
- data/lib/relaton/calconnect/hit_collection.rb +92 -0
- data/lib/relaton/calconnect/item_data.rb +12 -0
- data/lib/relaton/calconnect/model/bibdata.rb +8 -0
- data/lib/relaton/calconnect/model/bibitem.rb +8 -0
- data/lib/relaton/calconnect/model/doctype.rb +11 -0
- data/lib/relaton/calconnect/model/ext.rb +11 -0
- data/lib/relaton/calconnect/model/item.rb +20 -0
- data/lib/relaton/calconnect/processor.rb +72 -0
- data/lib/relaton/calconnect/scraper.rb +87 -0
- data/lib/relaton/calconnect/util.rb +8 -0
- data/lib/relaton/calconnect.rb +36 -0
- data/lib/relaton/ccsds/bibliography.rb +63 -0
- data/lib/relaton/ccsds/data/fetcher.rb +252 -0
- data/lib/relaton/ccsds/data/iso_references.rb +30 -0
- data/lib/relaton/ccsds/data/parser.rb +194 -0
- data/lib/relaton/ccsds/hit.rb +24 -0
- data/lib/relaton/ccsds/hit_collection.rb +47 -0
- data/lib/relaton/ccsds/item_data.rb +9 -0
- data/lib/relaton/ccsds/model/bibdata.rb +8 -0
- data/lib/relaton/ccsds/model/bibitem.rb +8 -0
- data/lib/relaton/ccsds/model/docidentifier.rb +121 -0
- data/lib/relaton/ccsds/model/doctype.rb +9 -0
- data/lib/relaton/ccsds/model/ext.rb +19 -0
- data/lib/relaton/ccsds/model/item.rb +15 -0
- data/lib/relaton/ccsds/processor.rb +68 -0
- data/lib/relaton/ccsds/util.rb +10 -0
- data/lib/relaton/ccsds.rb +33 -0
- data/lib/relaton/cen/bibliography.rb +149 -0
- data/lib/relaton/cen/committees.yaml +66 -0
- data/lib/relaton/cen/hit.rb +31 -0
- data/lib/relaton/cen/hit_collection.rb +116 -0
- data/lib/relaton/cen/item_data.rb +7 -0
- data/lib/relaton/cen/model/bibdata.rb +8 -0
- data/lib/relaton/cen/model/bibitem.rb +8 -0
- data/lib/relaton/cen/model/docidentifier.rb +100 -0
- data/lib/relaton/cen/model/ext.rb +11 -0
- data/lib/relaton/cen/model/item.rb +14 -0
- data/lib/relaton/cen/model/structured_identifier.rb +9 -0
- data/lib/relaton/cen/processor.rb +45 -0
- data/lib/relaton/cen/scraper.rb +218 -0
- data/lib/relaton/cen/util.rb +8 -0
- data/lib/relaton/cen.rb +30 -0
- data/lib/relaton/cie/bibdata.rb +8 -0
- data/lib/relaton/cie/bibitem.rb +8 -0
- data/lib/relaton/cie/bibliography.rb +31 -0
- data/lib/relaton/cie/data_fetcher.rb +540 -0
- data/lib/relaton/cie/ext.rb +7 -0
- data/lib/relaton/cie/item.rb +11 -0
- data/lib/relaton/cie/item_data.rb +6 -0
- data/lib/relaton/cie/processor.rb +68 -0
- data/lib/relaton/cie/scrapper.rb +52 -0
- data/lib/relaton/cie/util.rb +8 -0
- data/lib/relaton/cie.rb +29 -0
- data/lib/relaton/core/array_wrapper.rb +20 -0
- data/lib/relaton/core/data_fetcher.rb +244 -0
- data/lib/relaton/core/date_parser.rb +42 -0
- data/lib/relaton/core/governor.rb +320 -0
- data/lib/relaton/core/hash_keys_sybolizer.rb +19 -0
- data/lib/relaton/core/hit.rb +49 -0
- data/lib/relaton/core/hit_collection.rb +118 -0
- data/lib/relaton/core/pacer.rb +134 -0
- data/lib/relaton/core/processor.rb +67 -0
- data/lib/relaton/core/request_error.rb +14 -0
- data/lib/relaton/core/workers_pool.rb +45 -0
- data/lib/relaton/core.rb +12 -0
- data/lib/relaton/db/registry.rb +44 -4
- data/lib/relaton/db.rb +0 -1
- data/lib/relaton/doi/crossref.rb +89 -0
- data/lib/relaton/doi/parser.rb +921 -0
- data/lib/relaton/doi/processor.rb +65 -0
- data/lib/relaton/doi/util.rb +8 -0
- data/lib/relaton/doi.rb +20 -0
- data/lib/relaton/easc/bibdata.rb +8 -0
- data/lib/relaton/easc/bibitem.rb +8 -0
- data/lib/relaton/easc/bibliography.rb +95 -0
- data/lib/relaton/easc/docidentifier.rb +100 -0
- data/lib/relaton/easc/doctype.rb +14 -0
- data/lib/relaton/easc/ext.rb +44 -0
- data/lib/relaton/easc/item.rb +13 -0
- data/lib/relaton/easc/item_base.rb +18 -0
- data/lib/relaton/easc/item_data.rb +6 -0
- data/lib/relaton/easc/processor.rb +46 -0
- data/lib/relaton/easc/util.rb +8 -0
- data/lib/relaton/easc.rb +35 -0
- data/lib/relaton/ecma/bibdata.rb +8 -0
- data/lib/relaton/ecma/bibitem.rb +8 -0
- data/lib/relaton/ecma/bibliography.rb +149 -0
- data/lib/relaton/ecma/data_fetcher.rb +162 -0
- data/lib/relaton/ecma/data_parser.rb +49 -0
- data/lib/relaton/ecma/docidentifier.rb +124 -0
- data/lib/relaton/ecma/edition_parser.rb +80 -0
- data/lib/relaton/ecma/ext.rb +7 -0
- data/lib/relaton/ecma/item.rb +13 -0
- data/lib/relaton/ecma/item_data.rb +6 -0
- data/lib/relaton/ecma/memento_parser.rb +60 -0
- data/lib/relaton/ecma/page_fetcher.rb +39 -0
- data/lib/relaton/ecma/parser_common.rb +33 -0
- data/lib/relaton/ecma/processor.rb +69 -0
- data/lib/relaton/ecma/standard_parser.rb +134 -0
- data/lib/relaton/ecma/util.rb +8 -0
- data/lib/relaton/ecma.rb +33 -0
- data/lib/relaton/etsi/bibdata.rb +10 -0
- data/lib/relaton/etsi/bibitem.rb +10 -0
- data/lib/relaton/etsi/bibliography.rb +111 -0
- data/lib/relaton/etsi/data_fetcher.rb +167 -0
- data/lib/relaton/etsi/data_parser.rb +208 -0
- data/lib/relaton/etsi/doctype.rb +30 -0
- data/lib/relaton/etsi/ext.rb +31 -0
- data/lib/relaton/etsi/item.rb +15 -0
- data/lib/relaton/etsi/item_data.rb +6 -0
- data/lib/relaton/etsi/processor.rb +71 -0
- data/lib/relaton/etsi/pubid.rb +37 -0
- data/lib/relaton/etsi/status.rb +13 -0
- data/lib/relaton/etsi/util.rb +8 -0
- data/lib/relaton/etsi.rb +26 -0
- data/lib/relaton/gb/bibdata.rb +8 -0
- data/lib/relaton/gb/bibitem.rb +8 -0
- data/lib/relaton/gb/bibliography.rb +171 -0
- data/lib/relaton/gb/ccs.rb +14 -0
- data/lib/relaton/gb/committee.rb +13 -0
- data/lib/relaton/gb/docidentifier.rb +72 -0
- data/lib/relaton/gb/doctype.rb +9 -0
- data/lib/relaton/gb/ext.rb +36 -0
- data/lib/relaton/gb/gb_scraper.rb +61 -0
- data/lib/relaton/gb/gb_type.rb +20 -0
- data/lib/relaton/gb/hit.rb +48 -0
- data/lib/relaton/gb/hit_collection.rb +19 -0
- data/lib/relaton/gb/item.rb +13 -0
- data/lib/relaton/gb/item_data.rb +6 -0
- data/lib/relaton/gb/processor.rb +49 -0
- data/lib/relaton/gb/project_number.rb +38 -0
- data/lib/relaton/gb/scraper.rb +221 -0
- data/lib/relaton/gb/sec_scraper.rb +92 -0
- data/lib/relaton/gb/stage_name.rb +13 -0
- data/lib/relaton/gb/structured_identifier.rb +26 -0
- data/lib/relaton/gb/t_scraper.rb +126 -0
- data/lib/relaton/gb/util.rb +8 -0
- data/lib/relaton/gb/yaml/prefixes.yaml +200 -0
- data/lib/relaton/gb.rb +33 -0
- data/lib/relaton/gost/bibdata.rb +8 -0
- data/lib/relaton/gost/bibitem.rb +8 -0
- data/lib/relaton/gost/bibliography.rb +107 -0
- data/lib/relaton/gost/docidentifier.rb +80 -0
- data/lib/relaton/gost/doctype.rb +16 -0
- data/lib/relaton/gost/ext.rb +46 -0
- data/lib/relaton/gost/item.rb +15 -0
- data/lib/relaton/gost/item_base.rb +18 -0
- data/lib/relaton/gost/item_data.rb +6 -0
- data/lib/relaton/gost/processor.rb +49 -0
- data/lib/relaton/gost/util.rb +8 -0
- data/lib/relaton/gost.rb +36 -0
- data/lib/relaton/iala/bibdata.rb +8 -0
- data/lib/relaton/iala/bibitem.rb +8 -0
- data/lib/relaton/iala/bibliography.rb +146 -0
- data/lib/relaton/iala/docidentifier.rb +89 -0
- data/lib/relaton/iala/doctype.rb +18 -0
- data/lib/relaton/iala/ext.rb +32 -0
- data/lib/relaton/iala/item.rb +21 -0
- data/lib/relaton/iala/item_base.rb +18 -0
- data/lib/relaton/iala/item_data.rb +6 -0
- data/lib/relaton/iala/processor.rb +43 -0
- data/lib/relaton/iala/relation.rb +7 -0
- data/lib/relaton/iala/util.rb +8 -0
- data/lib/relaton/iala.rb +35 -0
- data/lib/relaton/iana/bibdata.rb +8 -0
- data/lib/relaton/iana/bibitem.rb +8 -0
- data/lib/relaton/iana/bibliography.rb +100 -0
- data/lib/relaton/iana/data_fetcher.rb +101 -0
- data/lib/relaton/iana/item.rb +7 -0
- data/lib/relaton/iana/item_data.rb +6 -0
- data/lib/relaton/iana/parser.rb +146 -0
- data/lib/relaton/iana/processor.rb +70 -0
- data/lib/relaton/iana/util.rb +8 -0
- data/lib/relaton/iana.rb +40 -0
- data/lib/relaton/iec/bibliography.rb +283 -0
- data/lib/relaton/iec/data_fetcher.rb +222 -0
- data/lib/relaton/iec/data_parser.rb +391 -0
- data/lib/relaton/iec/hit.rb +26 -0
- data/lib/relaton/iec/hit_collection.rb +138 -0
- data/lib/relaton/iec/item_data.rb +7 -0
- data/lib/relaton/iec/model/bibdata.rb +8 -0
- data/lib/relaton/iec/model/bibitem.rb +8 -0
- data/lib/relaton/iec/model/docidentifier.rb +135 -0
- data/lib/relaton/iec/model/doctype.rb +12 -0
- data/lib/relaton/iec/model/ext.rb +53 -0
- data/lib/relaton/iec/model/item.rb +20 -0
- data/lib/relaton/iec/model/item_base.rb +12 -0
- data/lib/relaton/iec/model/relation.rb +7 -0
- data/lib/relaton/iec/model/stage_name.rb +13 -0
- data/lib/relaton/iec/processor.rb +74 -0
- data/lib/relaton/iec/statuses.yml +199 -0
- data/lib/relaton/iec/util.rb +8 -0
- data/lib/relaton/iec.rb +98 -0
- data/lib/relaton/ieee/balloting_group.rb +13 -0
- data/lib/relaton/ieee/bibdata.rb +8 -0
- data/lib/relaton/ieee/bibitem.rb +8 -0
- data/lib/relaton/ieee/bibliography.rb +73 -0
- data/lib/relaton/ieee/converter/bibxml/from_rfcxml.rb +10 -0
- data/lib/relaton/ieee/converter/bibxml/from_rfcxml_referencegroup.rb +10 -0
- data/lib/relaton/ieee/converter/bibxml.rb +20 -0
- data/lib/relaton/ieee/data_fetcher.rb +771 -0
- data/lib/relaton/ieee/doctype.rb +9 -0
- data/lib/relaton/ieee/editorial_group.rb +19 -0
- data/lib/relaton/ieee/ext.rb +36 -0
- data/lib/relaton/ieee/idams_parser.rb +330 -0
- data/lib/relaton/ieee/item.rb +11 -0
- data/lib/relaton/ieee/item_data.rb +7 -0
- data/lib/relaton/ieee/processor.rb +71 -0
- data/lib/relaton/ieee/rawbib_id_parser.rb +753 -0
- data/lib/relaton/ieee/util.rb +8 -0
- data/lib/relaton/ieee.rb +30 -0
- data/lib/relaton/ietf/bibdata.rb +8 -0
- data/lib/relaton/ietf/bibitem.rb +8 -0
- data/lib/relaton/ietf/bibliography.rb +35 -0
- data/lib/relaton/ietf/bibxml_parser.rb +232 -0
- data/lib/relaton/ietf/data_fetcher.rb +561 -0
- data/lib/relaton/ietf/doctype.rb +9 -0
- data/lib/relaton/ietf/ext.rb +63 -0
- data/lib/relaton/ietf/item.rb +16 -0
- data/lib/relaton/ietf/item_base.rb +18 -0
- data/lib/relaton/ietf/item_data.rb +6 -0
- data/lib/relaton/ietf/processing_instructions.rb +79 -0
- data/lib/relaton/ietf/processor.rb +72 -0
- data/lib/relaton/ietf/relation.rb +9 -0
- data/lib/relaton/ietf/rfc/abstract.rb +19 -0
- data/lib/relaton/ietf/rfc/author.rb +21 -0
- data/lib/relaton/ietf/rfc/entry.rb +446 -0
- data/lib/relaton/ietf/rfc/entry_date.rb +21 -0
- data/lib/relaton/ietf/rfc/format.rb +19 -0
- data/lib/relaton/ietf/rfc/index.rb +46 -0
- data/lib/relaton/ietf/rfc/is_also.rb +21 -0
- data/lib/relaton/ietf/rfc/keywords.rb +19 -0
- data/lib/relaton/ietf/rfc/rfc_index_namespace.rb +11 -0
- data/lib/relaton/ietf/scraper.rb +101 -0
- data/lib/relaton/ietf/util.rb +8 -0
- data/lib/relaton/ietf/wg_name_resolver.rb +42 -0
- data/lib/relaton/ietf.rb +30 -0
- data/lib/relaton/iho/bibdata.rb +8 -0
- data/lib/relaton/iho/bibitem.rb +8 -0
- data/lib/relaton/iho/bibliography.rb +114 -0
- data/lib/relaton/iho/comment_period.rb +13 -0
- data/lib/relaton/iho/docidentifier.rb +31 -0
- data/lib/relaton/iho/doctype.rb +10 -0
- data/lib/relaton/iho/ext.rb +20 -0
- data/lib/relaton/iho/hash_parser_v1.rb +144 -0
- data/lib/relaton/iho/item.rb +20 -0
- data/lib/relaton/iho/item_base.rb +18 -0
- data/lib/relaton/iho/item_data.rb +6 -0
- data/lib/relaton/iho/processor.rb +54 -0
- data/lib/relaton/iho/relation.rb +9 -0
- data/lib/relaton/iho/structured_identifier.rb +20 -0
- data/lib/relaton/iho/util.rb +8 -0
- data/lib/relaton/iho.rb +27 -0
- data/lib/relaton/index/config.rb +52 -0
- data/lib/relaton/index/file_io.rb +305 -0
- data/lib/relaton/index/file_storage.rb +66 -0
- data/lib/relaton/index/pool.rb +46 -0
- data/lib/relaton/index/shard_source.rb +201 -0
- data/lib/relaton/index/type.rb +210 -0
- data/lib/relaton/index/util.rb +18 -0
- data/lib/relaton/index.rb +61 -0
- data/lib/relaton/isbn/isbn.rb +61 -0
- data/lib/relaton/isbn/open_library.rb +43 -0
- data/lib/relaton/isbn/parser.rb +88 -0
- data/lib/relaton/isbn/processor.rb +48 -0
- data/lib/relaton/isbn/util.rb +8 -0
- data/lib/relaton/isbn.rb +19 -0
- data/lib/relaton/iso/bibliography.rb +412 -0
- data/lib/relaton/iso/data_fetcher.rb +338 -0
- data/lib/relaton/iso/data_parser.rb +458 -0
- data/lib/relaton/iso/hash_parser_v1.rb +177 -0
- data/lib/relaton/iso/hit.rb +57 -0
- data/lib/relaton/iso/hit_collection.rb +196 -0
- data/lib/relaton/iso/item_data.rb +50 -0
- data/lib/relaton/iso/model/bibdata.rb +10 -0
- data/lib/relaton/iso/model/bibitem.rb +8 -0
- data/lib/relaton/iso/model/contributor.rb +6 -0
- data/lib/relaton/iso/model/contributor_info.rb +9 -0
- data/lib/relaton/iso/model/docidentifier.rb +146 -0
- data/lib/relaton/iso/model/doctype.rb +13 -0
- data/lib/relaton/iso/model/ext.rb +35 -0
- data/lib/relaton/iso/model/item.rb +17 -0
- data/lib/relaton/iso/model/item_base.rb +22 -0
- data/lib/relaton/iso/model/organization.rb +9 -0
- data/lib/relaton/iso/model/project_number.rb +22 -0
- data/lib/relaton/iso/model/relation.rb +9 -0
- data/lib/relaton/iso/model/stagename.rb +14 -0
- data/lib/relaton/iso/model/structured_identifier.rb +31 -0
- data/lib/relaton/iso/processor.rb +85 -0
- data/lib/relaton/iso/scraper.rb +635 -0
- data/lib/relaton/iso/type/pubid.rb +50 -0
- data/lib/relaton/iso/util.rb +8 -0
- data/lib/relaton/iso.rb +30 -0
- data/lib/relaton/itu/bibliography.rb +95 -0
- data/lib/relaton/itu/data_crawler_r.rb +664 -0
- data/lib/relaton/itu/data_fetcher.rb +550 -0
- data/lib/relaton/itu/data_merge_r.rb +149 -0
- data/lib/relaton/itu/data_parser_r.rb +214 -0
- data/lib/relaton/itu/data_parser_t.rb +228 -0
- data/lib/relaton/itu/family_cache.rb +177 -0
- data/lib/relaton/itu/governor.rb +56 -0
- data/lib/relaton/itu/hit.rb +28 -0
- data/lib/relaton/itu/hit_collection.rb +299 -0
- data/lib/relaton/itu/item_data.rb +6 -0
- data/lib/relaton/itu/model/approval_stage.rb +13 -0
- data/lib/relaton/itu/model/bibdata.rb +8 -0
- data/lib/relaton/itu/model/bibitem.rb +8 -0
- data/lib/relaton/itu/model/docidentifier.rb +75 -0
- data/lib/relaton/itu/model/doctype.rb +13 -0
- data/lib/relaton/itu/model/ext.rb +47 -0
- data/lib/relaton/itu/model/item.rb +13 -0
- data/lib/relaton/itu/model/meeting.rb +13 -0
- data/lib/relaton/itu/model/meeting_date.rb +15 -0
- data/lib/relaton/itu/model/question.rb +13 -0
- data/lib/relaton/itu/model/recommendation_status.rb +11 -0
- data/lib/relaton/itu/model/structured_identifier.rb +38 -0
- data/lib/relaton/itu/processor.rb +72 -0
- data/lib/relaton/itu/pubid.rb +199 -0
- data/lib/relaton/itu/radio_regulations_parser.rb +70 -0
- data/lib/relaton/itu/recommendation_fields.rb +334 -0
- data/lib/relaton/itu/recommendation_parser.rb +30 -0
- data/lib/relaton/itu/scraper.rb +203 -0
- data/lib/relaton/itu/util.rb +8 -0
- data/lib/relaton/itu.rb +34 -0
- data/lib/relaton/jcgm/bibdata.rb +8 -0
- data/lib/relaton/jcgm/bibitem.rb +8 -0
- data/lib/relaton/jcgm/bibliography.rb +97 -0
- data/lib/relaton/jcgm/data_fetcher.rb +81 -0
- data/lib/relaton/jcgm/docidentifier.rb +102 -0
- data/lib/relaton/jcgm/doctype.rb +12 -0
- data/lib/relaton/jcgm/ext.rb +23 -0
- data/lib/relaton/jcgm/item.rb +20 -0
- data/lib/relaton/jcgm/item_base.rb +18 -0
- data/lib/relaton/jcgm/item_data.rb +6 -0
- data/lib/relaton/jcgm/meetings_parser.rb +175 -0
- data/lib/relaton/jcgm/processor.rb +71 -0
- data/lib/relaton/jcgm/relation.rb +9 -0
- data/lib/relaton/jcgm/structured_identifier.rb +40 -0
- data/lib/relaton/jcgm/util.rb +8 -0
- data/lib/relaton/jcgm.rb +24 -0
- data/lib/relaton/jis/bibdata.rb +8 -0
- data/lib/relaton/jis/bibitem.rb +8 -0
- data/lib/relaton/jis/bibliography.rb +73 -0
- data/lib/relaton/jis/data_fetcher.rb +183 -0
- data/lib/relaton/jis/docidentifier.rb +120 -0
- data/lib/relaton/jis/doctype.rb +9 -0
- data/lib/relaton/jis/ext.rb +13 -0
- data/lib/relaton/jis/hit.rb +88 -0
- data/lib/relaton/jis/hit_collection.rb +128 -0
- data/lib/relaton/jis/item.rb +22 -0
- data/lib/relaton/jis/item_base.rb +26 -0
- data/lib/relaton/jis/item_data.rb +8 -0
- data/lib/relaton/jis/processor.rb +55 -0
- data/lib/relaton/jis/relation.rb +11 -0
- data/lib/relaton/jis/scraper.rb +232 -0
- data/lib/relaton/jis/structured_identifier.rb +13 -0
- data/lib/relaton/jis/util.rb +8 -0
- data/lib/relaton/jis.rb +32 -0
- data/lib/relaton/logger/channels/gh_issue.rb +143 -0
- data/lib/relaton/logger/config.rb +34 -0
- data/lib/relaton/logger/formatter_json.rb +8 -0
- data/lib/relaton/logger/formatter_string.rb +12 -0
- data/lib/relaton/logger/log.rb +64 -0
- data/lib/relaton/logger/log_device.rb +27 -0
- data/lib/relaton/logger/pool.rb +24 -0
- data/lib/relaton/logger.rb +18 -0
- data/lib/relaton/nist/bibdata.rb +8 -0
- data/lib/relaton/nist/bibitem.rb +8 -0
- data/lib/relaton/nist/bibliography.rb +191 -0
- data/lib/relaton/nist/comment_period.rb +15 -0
- data/lib/relaton/nist/data_fetcher.rb +134 -0
- data/lib/relaton/nist/date.rb +7 -0
- data/lib/relaton/nist/docidentifier.rb +165 -0
- data/lib/relaton/nist/doctype.rb +7 -0
- data/lib/relaton/nist/ext.rb +16 -0
- data/lib/relaton/nist/hit.rb +18 -0
- data/lib/relaton/nist/hit_collection.rb +362 -0
- data/lib/relaton/nist/item.rb +19 -0
- data/lib/relaton/nist/item_base.rb +16 -0
- data/lib/relaton/nist/item_data.rb +6 -0
- data/lib/relaton/nist/mods_parser.rb +302 -0
- data/lib/relaton/nist/processor.rb +69 -0
- data/lib/relaton/nist/pubs_export.rb +69 -0
- data/lib/relaton/nist/relation.rb +10 -0
- data/lib/relaton/nist/scraper.rb +293 -0
- data/lib/relaton/nist/series.yaml +49 -0
- data/lib/relaton/nist/util.rb +8 -0
- data/lib/relaton/nist.rb +34 -0
- data/lib/relaton/oasis/bibdata.rb +8 -0
- data/lib/relaton/oasis/bibitem.rb +8 -0
- data/lib/relaton/oasis/bibliography.rb +201 -0
- data/lib/relaton/oasis/browser_agent.rb +71 -0
- data/lib/relaton/oasis/data_fetcher.rb +130 -0
- data/lib/relaton/oasis/data_parser.rb +305 -0
- data/lib/relaton/oasis/data_parser_utils.rb +330 -0
- data/lib/relaton/oasis/data_part_parser.rb +239 -0
- data/lib/relaton/oasis/docidentifier.rb +54 -0
- data/lib/relaton/oasis/doctype.rb +7 -0
- data/lib/relaton/oasis/ext.rb +19 -0
- data/lib/relaton/oasis/item.rb +14 -0
- data/lib/relaton/oasis/item_data.rb +6 -0
- data/lib/relaton/oasis/processor.rb +74 -0
- data/lib/relaton/oasis/util.rb +8 -0
- data/lib/relaton/oasis.rb +41 -0
- data/lib/relaton/ogc/bibdata.rb +8 -0
- data/lib/relaton/ogc/bibitem.rb +8 -0
- data/lib/relaton/ogc/bibliography.rb +64 -0
- data/lib/relaton/ogc/data_fetcher.rb +131 -0
- data/lib/relaton/ogc/docidentifier.rb +111 -0
- data/lib/relaton/ogc/doctype.rb +12 -0
- data/lib/relaton/ogc/ext.rb +15 -0
- data/lib/relaton/ogc/hit.rb +18 -0
- data/lib/relaton/ogc/hit_collection.rb +111 -0
- data/lib/relaton/ogc/item.rb +13 -0
- data/lib/relaton/ogc/item_data.rb +6 -0
- data/lib/relaton/ogc/processor.rb +59 -0
- data/lib/relaton/ogc/scraper.rb +225 -0
- data/lib/relaton/ogc/util.rb +8 -0
- data/lib/relaton/ogc.rb +30 -0
- data/lib/relaton/oiml/bibdata.rb +8 -0
- data/lib/relaton/oiml/bibitem.rb +8 -0
- data/lib/relaton/oiml/bibliography.rb +169 -0
- data/lib/relaton/oiml/docidentifier.rb +48 -0
- data/lib/relaton/oiml/doctype.rb +13 -0
- data/lib/relaton/oiml/ext.rb +42 -0
- data/lib/relaton/oiml/item.rb +20 -0
- data/lib/relaton/oiml/item_base.rb +18 -0
- data/lib/relaton/oiml/item_data.rb +6 -0
- data/lib/relaton/oiml/processor.rb +54 -0
- data/lib/relaton/oiml/relation.rb +9 -0
- data/lib/relaton/oiml/util.rb +8 -0
- data/lib/relaton/oiml.rb +24 -0
- data/lib/relaton/omg/bibdata.rb +8 -0
- data/lib/relaton/omg/bibitem.rb +8 -0
- data/lib/relaton/omg/bibliography.rb +31 -0
- data/lib/relaton/omg/docidentifier.rb +67 -0
- data/lib/relaton/omg/ext.rb +7 -0
- data/lib/relaton/omg/item.rb +9 -0
- data/lib/relaton/omg/item_data.rb +6 -0
- data/lib/relaton/omg/processor.rb +35 -0
- data/lib/relaton/omg/scraper.rb +186 -0
- data/lib/relaton/omg/util.rb +8 -0
- data/lib/relaton/omg.rb +25 -0
- data/lib/relaton/plateau/bibdata.rb +8 -0
- data/lib/relaton/plateau/bibitem.rb +8 -0
- data/lib/relaton/plateau/bibliography.rb +31 -0
- data/lib/relaton/plateau/data_fetcher.rb +176 -0
- data/lib/relaton/plateau/doctype.rb +7 -0
- data/lib/relaton/plateau/ext.rb +23 -0
- data/lib/relaton/plateau/handbook_parser.rb +102 -0
- data/lib/relaton/plateau/hit.rb +21 -0
- data/lib/relaton/plateau/hit_collection.rb +79 -0
- data/lib/relaton/plateau/item.rb +12 -0
- data/lib/relaton/plateau/item_data.rb +6 -0
- data/lib/relaton/plateau/parser.rb +101 -0
- data/lib/relaton/plateau/processor.rb +51 -0
- data/lib/relaton/plateau/technical_report_parser.rb +98 -0
- data/lib/relaton/plateau/util.rb +8 -0
- data/lib/relaton/plateau.rb +29 -0
- data/lib/relaton/sdo/config.rb +34 -0
- data/lib/relaton/sdo/fetcher.rb +52 -0
- data/lib/relaton/sdo/logo.rb +95 -0
- data/lib/relaton/sdo/name.rb +26 -0
- data/lib/relaton/sdo/organization.rb +71 -0
- data/lib/relaton/sdo/store.rb +49 -0
- data/lib/relaton/sdo.rb +29 -0
- data/lib/relaton/un/bibdata.rb +8 -0
- data/lib/relaton/un/bibitem.rb +8 -0
- data/lib/relaton/un/bibliography.rb +48 -0
- data/lib/relaton/un/doctype.rb +10 -0
- data/lib/relaton/un/ext.rb +30 -0
- data/lib/relaton/un/hit.rb +56 -0
- data/lib/relaton/un/hit_collection.rb +61 -0
- data/lib/relaton/un/item.rb +11 -0
- data/lib/relaton/un/item_data.rb +6 -0
- data/lib/relaton/un/parser.rb +108 -0
- data/lib/relaton/un/processor.rb +38 -0
- data/lib/relaton/un/session.rb +25 -0
- data/lib/relaton/un/token_generator.rb +105 -0
- data/lib/relaton/un/util.rb +8 -0
- data/lib/relaton/un/wasm/decoder.rb +434 -0
- data/lib/relaton/un/wasm/instance.rb +101 -0
- data/lib/relaton/un/wasm/interpreter.rb +613 -0
- data/lib/relaton/un/wasm/memory.rb +112 -0
- data/lib/relaton/un/wasm/module.rb +47 -0
- data/lib/relaton/un/wasm.rb +17 -0
- data/lib/relaton/un/wasm_v_bg.wasm +0 -0
- data/lib/relaton/un.rb +28 -0
- data/lib/relaton/version.rb +1 -1
- data/lib/relaton/w3c/bibdata.rb +8 -0
- data/lib/relaton/w3c/bibitem.rb +8 -0
- data/lib/relaton/w3c/bibliography.rb +174 -0
- data/lib/relaton/w3c/data_fetcher.rb +487 -0
- data/lib/relaton/w3c/data_parser.rb +379 -0
- data/lib/relaton/w3c/docidentifier.rb +48 -0
- data/lib/relaton/w3c/doctype.rb +7 -0
- data/lib/relaton/w3c/ext.rb +11 -0
- data/lib/relaton/w3c/governor.rb +32 -0
- data/lib/relaton/w3c/item.rb +15 -0
- data/lib/relaton/w3c/item_data.rb +6 -0
- data/lib/relaton/w3c/processor.rb +49 -0
- data/lib/relaton/w3c/pubid.rb +85 -0
- data/lib/relaton/w3c/safe_realize.rb +148 -0
- data/lib/relaton/w3c/util.rb +8 -0
- data/lib/relaton/w3c.rb +36 -0
- data/lib/relaton/xsf/bibdata.rb +8 -0
- data/lib/relaton/xsf/bibitem.rb +8 -0
- data/lib/relaton/xsf/bibliography.rb +84 -0
- data/lib/relaton/xsf/data_fetcher.rb +106 -0
- data/lib/relaton/xsf/docidentifier.rb +46 -0
- data/lib/relaton/xsf/hit.rb +17 -0
- data/lib/relaton/xsf/hit_collection.rb +52 -0
- data/lib/relaton/xsf/item.rb +13 -0
- data/lib/relaton/xsf/item_data.rb +6 -0
- data/lib/relaton/xsf/processor.rb +51 -0
- data/lib/relaton/xsf/util.rb +9 -0
- data/lib/relaton/xsf.rb +34 -0
- data/lib/relaton-core.rb +1 -0
- data/lib/relaton.rb +85 -0
- metadata +775 -161
- data/.github/workflows/rake.yml +0 -14
- data/.github/workflows/release.yml +0 -24
- data/.github/workflows/rubocop.yml +0 -48
- data/.gitignore +0 -12
- data/.rspec +0 -3
- data/CLAUDE.md +0 -78
- data/Gemfile +0 -29
- data/Rakefile +0 -6
- data/docs/CHANGELOG.adoc +0 -16
- data/docs/VERSIONING_POLICY.adoc +0 -38
- data/docs/navigation.adoc +0 -6
- data/lib/relaton/db/version.rb +0 -5
- data/relaton.gemspec +0 -72
- data/spec/relaton/config_spec.rb +0 -10
- data/spec/relaton/db_cache_spec.rb +0 -51
- data/spec/relaton/db_spec.rb +0 -567
- data/spec/relaton/registry_spec.rb +0 -178
- data/spec/relaton/util_spec.rb +0 -3
- data/spec/relaton_meta_spec.rb +0 -25
- data/spec/relaton_spec.rb +0 -800
- data/spec/spec_helper.rb +0 -45
- data/spec/support/gb_t_20223_2006.xml +0 -33
- data/spec/support/iso_111111119115_1.xml +0 -0
- data/spec/support/iso_19115_1.xml +0 -115
- data/spec/support/iso_19115_2.xml +0 -95
- data/spec/support/rfc_8341.xml +0 -46
- data/spec/vcr_cassetes/api_relaton_org.yml +0 -121
- data/spec/vcr_cassetes/api_relaton_org_unavailable.yml +0 -140
- data/spec/vcr_cassetes/cc_dir_10005_2019.yml +0 -202
- data/spec/vcr_cassetes/cipm_meeting_43.yml +0 -1505
- data/spec/vcr_cassetes/gb_t_20223_2006.yml +0 -729
- data/spec/vcr_cassetes/iso_111111119115_1.yml +0 -12951
- data/spec/vcr_cassetes/iso_19115_1.yml +0 -17334
- data/spec/vcr_cassetes/iso_19115_1_2.yml +0 -322
- data/spec/vcr_cassetes/iso_19115_1_std.yml +0 -17334
- data/spec/vcr_cassetes/iso_19115_all_parts.yml +0 -185
- data/spec/vcr_cassetes/iso_19133_2005.yml +0 -144
- data/spec/vcr_cassetes/iso_combined_applied.yml +0 -318
- data/spec/vcr_cassetes/iso_combined_included.yml +0 -318
- data/spec/vcr_cassetes/ogc_19_025r1.yml +0 -374
- data/spec/vcr_cassetes/omg_ami4ccm_1_0.yml +0 -317
- data/spec/vcr_cassetes/rfc_8341.yml +0 -1278
data/lib/relaton/cie.rb
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
require "nokogiri"
|
|
2
|
+
# require "parslet"
|
|
3
|
+
require "pubid"
|
|
4
|
+
require "relaton/index"
|
|
5
|
+
require "relaton/bib"
|
|
6
|
+
# require "relaton_bib/name_parser"
|
|
7
|
+
require_relative "version"
|
|
8
|
+
require_relative "cie/util"
|
|
9
|
+
require_relative "cie/item_data"
|
|
10
|
+
require_relative "cie/item"
|
|
11
|
+
require_relative "cie/bibitem"
|
|
12
|
+
require_relative "cie/bibdata"
|
|
13
|
+
require_relative "cie/bibliography"
|
|
14
|
+
require_relative "cie/scrapper"
|
|
15
|
+
|
|
16
|
+
module Relaton
|
|
17
|
+
module Cie
|
|
18
|
+
INDEXFILE = "index-v2".freeze
|
|
19
|
+
|
|
20
|
+
# Returns hash of XML reammar
|
|
21
|
+
# @return [String]
|
|
22
|
+
def self.grammar_hash
|
|
23
|
+
# gem_path = File.expand_path "..", __dir__
|
|
24
|
+
# grammars_path = File.join gem_path, "grammars", "*"
|
|
25
|
+
# grammars = Dir[grammars_path].sort.map { |gp| File.read gp }.join
|
|
26
|
+
Digest::MD5.hexdigest Relaton::VERSION # grammars
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
end
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
module Relaton
|
|
2
|
+
module Core
|
|
3
|
+
module ArrayWrapper
|
|
4
|
+
#
|
|
5
|
+
# Wrap into Array if not Array
|
|
6
|
+
#
|
|
7
|
+
# @param [Object] content
|
|
8
|
+
#
|
|
9
|
+
# @return [Array<Object>]
|
|
10
|
+
#
|
|
11
|
+
def array(content)
|
|
12
|
+
case content
|
|
13
|
+
when Array then content
|
|
14
|
+
when nil then []
|
|
15
|
+
else [content]
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
end
|
|
@@ -0,0 +1,244 @@
|
|
|
1
|
+
require "fileutils"
|
|
2
|
+
|
|
3
|
+
module Relaton
|
|
4
|
+
module Core
|
|
5
|
+
class DataFetcher
|
|
6
|
+
# attr_accessor :docs
|
|
7
|
+
#
|
|
8
|
+
# Initialize fetcher
|
|
9
|
+
#
|
|
10
|
+
# @param [String] output path to output directory
|
|
11
|
+
# @param [String] format output format (yaml, xml, bibxml)
|
|
12
|
+
#
|
|
13
|
+
def initialize(output, format)
|
|
14
|
+
@output = output
|
|
15
|
+
@format = format
|
|
16
|
+
@ext = format.sub "bibxml", "xml"
|
|
17
|
+
@files = Set.new
|
|
18
|
+
# path => docid that reserved it, for #unique_output_file. Distinct
|
|
19
|
+
# from @files, which flavors use for their own duplicate handling:
|
|
20
|
+
# this one has to know WHICH document owns a path, not just that it
|
|
21
|
+
# is taken.
|
|
22
|
+
@file_docids = {}
|
|
23
|
+
# Paths this process wrote during this run; see #write_unique.
|
|
24
|
+
@written = Set.new
|
|
25
|
+
# Set true by a fetcher that writes from forked worker processes,
|
|
26
|
+
# where @file_docids cannot see a peer's claim. See #write_unique.
|
|
27
|
+
@cross_process = false
|
|
28
|
+
# @docs = []
|
|
29
|
+
@errors = Hash.new(true)
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# API method for external service
|
|
33
|
+
#
|
|
34
|
+
# @return the value returned by the instance `#fetch`, so callers can act
|
|
35
|
+
# on the outcome (e.g. relaton-iso returns whether it rebuilt).
|
|
36
|
+
def self.fetch(source = nil, output: "data", format: "yaml")
|
|
37
|
+
t1 = Time.now
|
|
38
|
+
puts "Started at: #{t1}"
|
|
39
|
+
FileUtils.mkdir_p output
|
|
40
|
+
result = new(output, format).fetch(source)
|
|
41
|
+
t2 = Time.now
|
|
42
|
+
puts "Stopped at: #{t2}"
|
|
43
|
+
puts "Done in: #{(t2 - t1).round} sec."
|
|
44
|
+
result
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def fetch(source = nil)
|
|
48
|
+
raise NotImplementedError, "#{self.class}#fetch method must be implemented"
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def gh_issue
|
|
52
|
+
return @gh_issue if defined? @gh_issue
|
|
53
|
+
|
|
54
|
+
channel = gh_issue_channel
|
|
55
|
+
if channel[0]
|
|
56
|
+
@gh_issue = Relaton::Logger::Channels::GhIssue.new(*channel)
|
|
57
|
+
Relaton.logger_pool[:gh_issue] = Relaton::Logger::Log.new(@gh_issue, levels: [:error])
|
|
58
|
+
end
|
|
59
|
+
@gh_issue
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def gh_issue_channel
|
|
63
|
+
[ENV.fetch("GITHUB_REPOSITORY", nil), "Error fetching documents"]
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def report_errors
|
|
67
|
+
gh_issue # register the channel before logging
|
|
68
|
+
@errors.select { |_, v| v }.each do |key, value|
|
|
69
|
+
# A String value IS the message: a specific, per-document failure
|
|
70
|
+
# such as an unparseable identifier, reported through this same
|
|
71
|
+
# channel without needing a per-flavor override. A boolean means
|
|
72
|
+
# "this field failed for every record" — what the flavors' own
|
|
73
|
+
# ERROR_KEYS track — and its message is derived from the key.
|
|
74
|
+
log_error value.is_a?(String) ? value : "Failed to fetch #{key}"
|
|
75
|
+
end
|
|
76
|
+
@gh_issue&.create_issue
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def log_error(_msg)
|
|
80
|
+
raise NoMatchingPatternError, "#{self.class}#log_error method must be implemented"
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
# Most filesystems cap a single path component at 255 bytes.
|
|
84
|
+
MAX_BASENAME_BYTES = 255
|
|
85
|
+
|
|
86
|
+
# Create-or-fail. The failure is the point: it is how one process learns
|
|
87
|
+
# that another already holds a path. See #write_unique.
|
|
88
|
+
EXCLUSIVE = File::WRONLY | File::CREAT | File::EXCL
|
|
89
|
+
|
|
90
|
+
# @param [String] document ID
|
|
91
|
+
# @return [String] filename based on PubID identifier
|
|
92
|
+
#
|
|
93
|
+
# A docid can be pathologically long (pubid's `to_s` for amendment docs
|
|
94
|
+
# embeds the full "(Amendment to … as amended by …)" clause), which would
|
|
95
|
+
# make the basename exceed the OS limit and raise Errno::ENAMETOOLONG on
|
|
96
|
+
# write. When that happens, truncate the sanitized id and append a short
|
|
97
|
+
# digest of the full docid so the filename stays bounded, unique, and
|
|
98
|
+
# deterministic (every call site round-trips through this method).
|
|
99
|
+
def output_file(docid)
|
|
100
|
+
id = docid.downcase.gsub(/[.,\s\/:()-]+/, "-").delete_suffix("-")
|
|
101
|
+
ext = ".#{@ext}"
|
|
102
|
+
limit = MAX_BASENAME_BYTES - ext.bytesize
|
|
103
|
+
if id.bytesize > limit
|
|
104
|
+
require "digest"
|
|
105
|
+
suffix = "-#{Digest::SHA1.hexdigest(docid)[0, 12]}"
|
|
106
|
+
# `id` has no consecutive "-" (gsub collapsed runs above), so
|
|
107
|
+
# truncation leaves at most one trailing "-" — delete_suffix is
|
|
108
|
+
# enough and avoids a polynomial-ReDoS regex on the docid.
|
|
109
|
+
id = id.byteslice(0, limit - suffix.bytesize).scrub("").delete_suffix("-") + suffix
|
|
110
|
+
end
|
|
111
|
+
File.join @output, "#{id}#{ext}"
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
#
|
|
115
|
+
# Reserve a unique output path for `docid`.
|
|
116
|
+
#
|
|
117
|
+
# `output_file` sanitizes ".", ",", "/", ":", "(", ")", "-" and whitespace
|
|
118
|
+
# all to "-", so two DISTINCT docids can map to one path — live instance in
|
|
119
|
+
# relaton-data-iana: `rpki/signed-objects` and `rpki-signed-objects` both
|
|
120
|
+
# give `data/rpki-signed-objects.yaml`. A caller that merely warns and
|
|
121
|
+
# writes anyway leaves one file holding the wrong document for one of two
|
|
122
|
+
# index ids: a wrong answer, not a missing one.
|
|
123
|
+
#
|
|
124
|
+
# Returns `output_file(docid)` when that path is free, or when it is
|
|
125
|
+
# already held by this SAME docid — a genuine duplicate is the caller's
|
|
126
|
+
# business (skip / merge / last-wins), and this method must not turn one
|
|
127
|
+
# into two files. Only a real clash with a DIFFERENT docid gets a variant,
|
|
128
|
+
# suffixed with a digest of this docid so the name depends on the document
|
|
129
|
+
# and not on encounter order: adding a record never renames an existing
|
|
130
|
+
# file. (Which member of a clashing pair keeps the plain path does follow
|
|
131
|
+
# write order, which is stable for a given corpus.)
|
|
132
|
+
#
|
|
133
|
+
# @param [String] docid
|
|
134
|
+
# @return [String] path, reserved for this docid
|
|
135
|
+
#
|
|
136
|
+
def unique_output_file(docid)
|
|
137
|
+
file = output_file docid
|
|
138
|
+
owner = @file_docids[file]
|
|
139
|
+
file = digest_output_file(docid) unless owner.nil? || owner == docid
|
|
140
|
+
reserve file, docid
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
#
|
|
144
|
+
# Write `content` for `docid`, never clobbering a different document.
|
|
145
|
+
#
|
|
146
|
+
# `output_file` is not injective, so the plain path may belong to someone
|
|
147
|
+
# else. `unique_output_file` settles that within this process; the file is
|
|
148
|
+
# then created with `O_EXCL` so a peer *process* cannot be clobbered
|
|
149
|
+
# either. Returns the path actually written.
|
|
150
|
+
#
|
|
151
|
+
# The four outcomes, each load-bearing:
|
|
152
|
+
#
|
|
153
|
+
# * **We already wrote this path for this docid** — plain overwrite, so a
|
|
154
|
+
# genuine duplicate stays one file and the flavor's own duplicate
|
|
155
|
+
# handling (skip / merge / last-wins) still decides what happens.
|
|
156
|
+
# * **`O_EXCL` succeeds** — the normal case.
|
|
157
|
+
# * **`EEXIST`, single process** — `unique_output_file` has already proved
|
|
158
|
+
# that no peer of ours holds this path, so the file can only be a
|
|
159
|
+
# leftover from an earlier crawl. Overwrite it. Every single-process
|
|
160
|
+
# flavor takes this branch, and it involves no guesswork.
|
|
161
|
+
# * **`EEXIST`, `@cross_process`** — a peer's file and a leftover are
|
|
162
|
+
# indistinguishable, so take a path of our own rather than risk
|
|
163
|
+
# destroying a record. The caller reconciles the names afterwards, in
|
|
164
|
+
# the parent, where it can see every docid (see
|
|
165
|
+
# `Relaton::Ietf::DataFetcher#reconcile_output_files`).
|
|
166
|
+
#
|
|
167
|
+
# Deliberately NOT a wall-clock "is this file older than the crawl?" test.
|
|
168
|
+
# A crawl runs into a populated `data/`, so `EEXIST` is the common case on
|
|
169
|
+
# a re-run and such a test would decide it for every record; its
|
|
170
|
+
# false-"stale" direction is the silent overwrite this method exists to
|
|
171
|
+
# prevent, and coarse filesystem mtime granularity makes that reachable.
|
|
172
|
+
#
|
|
173
|
+
# @param [String] docid
|
|
174
|
+
# @param [String] content serialized document
|
|
175
|
+
# @return [String] path written
|
|
176
|
+
#
|
|
177
|
+
def write_unique(docid, content)
|
|
178
|
+
file = unique_output_file docid
|
|
179
|
+
return force_write(file, content) if @written.include?(file)
|
|
180
|
+
|
|
181
|
+
File.write file, content, mode: EXCLUSIVE, encoding: "UTF-8"
|
|
182
|
+
@written << file
|
|
183
|
+
file
|
|
184
|
+
rescue Errno::EEXIST
|
|
185
|
+
return force_write(file, content) unless @cross_process
|
|
186
|
+
|
|
187
|
+
force_write reserve(digest_output_file(docid), docid), content
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
#
|
|
191
|
+
# Serialize bibliographic item
|
|
192
|
+
#
|
|
193
|
+
# @param [RelatonCcsds::BibliographicItem] bib <description>
|
|
194
|
+
#
|
|
195
|
+
# @return [String] serialized bibliographic item
|
|
196
|
+
#
|
|
197
|
+
def serialize(bib)
|
|
198
|
+
send "to_#{@format}", bib
|
|
199
|
+
end
|
|
200
|
+
|
|
201
|
+
def to_yaml(bib)
|
|
202
|
+
raise NotImplementedError, "#{self.class}#to_yaml method must be implemented"
|
|
203
|
+
end
|
|
204
|
+
|
|
205
|
+
def to_xml(bib)
|
|
206
|
+
raise NotImplementedError, "#{self.class}#to_xml method must be implemented"
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
def to_bibxml(bib)
|
|
210
|
+
raise NotImplementedError, "#{self.class}#to_bibxml method must be implemented"
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
private
|
|
214
|
+
|
|
215
|
+
# Write, clobbering whatever is there. The caller has already established
|
|
216
|
+
# that the path is ours to take.
|
|
217
|
+
def force_write(file, content)
|
|
218
|
+
File.write file, content, encoding: "UTF-8"
|
|
219
|
+
@written << file
|
|
220
|
+
file
|
|
221
|
+
end
|
|
222
|
+
|
|
223
|
+
# Record `docid` as the owner of `file`, unless someone got there first,
|
|
224
|
+
# and return `file`. The reservation table must never hand one path to two
|
|
225
|
+
# documents; see #unique_output_file.
|
|
226
|
+
def reserve(file, docid)
|
|
227
|
+
@file_docids[file] ||= docid
|
|
228
|
+
file
|
|
229
|
+
end
|
|
230
|
+
|
|
231
|
+
# `output_file`'s path with a docid digest appended to the basename, kept
|
|
232
|
+
# inside the same byte cap. Deterministic: same docid, same name.
|
|
233
|
+
def digest_output_file(docid)
|
|
234
|
+
require "digest"
|
|
235
|
+
ext = ".#{@ext}"
|
|
236
|
+
suffix = "-#{Digest::SHA1.hexdigest(docid)[0, 12]}"
|
|
237
|
+
stem = File.basename output_file(docid), ext
|
|
238
|
+
limit = MAX_BASENAME_BYTES - ext.bytesize - suffix.bytesize
|
|
239
|
+
stem = stem.byteslice(0, limit).scrub("").delete_suffix("-") if stem.bytesize > limit
|
|
240
|
+
File.join @output, "#{stem}#{suffix}#{ext}"
|
|
241
|
+
end
|
|
242
|
+
end
|
|
243
|
+
end
|
|
244
|
+
end
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
module Relaton
|
|
2
|
+
module Core
|
|
3
|
+
module DateParser
|
|
4
|
+
# @param date [String, Integer, Date] date
|
|
5
|
+
# @param str [Boolean] return string or Date
|
|
6
|
+
# @return [Date, String, nil] date
|
|
7
|
+
def parse_date(date, str: true) # rubocop:disable Metrics/CyclomaticComplexity,Metrics/MethodLength
|
|
8
|
+
return date if date.is_a?(Date)
|
|
9
|
+
|
|
10
|
+
case date.to_s
|
|
11
|
+
when /(?<date>\w+\s\d{4})/ # February 2012
|
|
12
|
+
format_date $~[:date], "%B %Y", str, "%Y-%m"
|
|
13
|
+
when /(?<date>\w+\s\d{1,2},\s\d{4})/ # February 11, 2012
|
|
14
|
+
format_date $~[:date], "%B %d, %Y", str, "%Y-%m-%d"
|
|
15
|
+
when /(?<date>\d{4}-\d{1,2}-\d{1,2})/ # 2012-02-03 or 2012-2-3
|
|
16
|
+
format_date $~[:date], "%Y-%m-%d", str
|
|
17
|
+
when /(?<date>\d{4}-\d{1,2})/ # 2012-02 or 2012-2
|
|
18
|
+
format_date $~[:date], "%Y-%m", str
|
|
19
|
+
when /(?<date>\d{4})/ # 2012
|
|
20
|
+
format_date $~[:date], "%Y", str
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
private
|
|
25
|
+
|
|
26
|
+
#
|
|
27
|
+
# Parse date string to Date object and format it
|
|
28
|
+
#
|
|
29
|
+
# @param [String] date date string
|
|
30
|
+
# @param [String] format format string
|
|
31
|
+
# @param [Boolean] str return string if true in other case return Date
|
|
32
|
+
# @param [String, nil] outformat output format
|
|
33
|
+
#
|
|
34
|
+
# @return [Date, String] date object or formatted date string
|
|
35
|
+
#
|
|
36
|
+
def format_date(date, format, str, outformat = nil)
|
|
37
|
+
date = Date.strptime(date, format)
|
|
38
|
+
str ? date.strftime(outformat || format) : date
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
end
|
|
@@ -0,0 +1,320 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "time" # Time.parse, for an HTTP-date Retry-After
|
|
4
|
+
|
|
5
|
+
module Relaton
|
|
6
|
+
module Core
|
|
7
|
+
#
|
|
8
|
+
# Process-wide, thread-safe rate-limit back-pressure for a crawl.
|
|
9
|
+
#
|
|
10
|
+
# Promoted here from `Relaton::W3c::Governor`, which is now a thin binding
|
|
11
|
+
# of it, when the ITU-R crawler became the second flavor to need
|
|
12
|
+
# cross-thread throttling. What stays with a flavor is only which errors
|
|
13
|
+
# mean "throttled" (THROTTLE_ERRORS / .throttle?) and which env vars tune it
|
|
14
|
+
# (ENV_PREFIX); the ladder, the jitter, the give-up latch and the
|
|
15
|
+
# Retry-After parsing are shared.
|
|
16
|
+
#
|
|
17
|
+
# The problem it solves, as first met on api.w3.org: that host sits behind
|
|
18
|
+
# Cloudflare, whose rate limiting is bimodal — a run either never trips it
|
|
19
|
+
# or trips it and stays banned for hours. Before this
|
|
20
|
+
# existed, each of the crawl's workers reacted to its own 429s in isolation
|
|
21
|
+
# with a backoff capped at ~31s — far too short to outlast the ban, and the
|
|
22
|
+
# retries themselves kept the limiter engaged. The Aug-2026 crawl spent 4h56m
|
|
23
|
+
# doing nothing but re-tripping the limiter, permanently discarding 1,412
|
|
24
|
+
# documents, before it aborted.
|
|
25
|
+
#
|
|
26
|
+
# The governor gives the whole worker pool one shared cooldown. When any
|
|
27
|
+
# thread sees a 429 it opens (or extends) that cooldown; every other thread
|
|
28
|
+
# observes it in #wait. Cooldowns escalate per *round* — 60s, 120s, ... to a
|
|
29
|
+
# 15-minute cap — and after GIVE_UP_AFTER consecutive rounds without a single
|
|
30
|
+
# success the crawl is declared rate-limited and gives up (~15 min) so CI
|
|
31
|
+
# fails fast instead of grinding for five hours.
|
|
32
|
+
#
|
|
33
|
+
# `clock:` and `sleeper:` are injectable so specs exercise the real
|
|
34
|
+
# arithmetic without wall-clock waits.
|
|
35
|
+
#
|
|
36
|
+
class Governor
|
|
37
|
+
# First cooldown, in seconds. Deliberately minute-scale: a Cloudflare ban
|
|
38
|
+
# window outlasts any second-scale backoff.
|
|
39
|
+
BASE_COOLDOWN = 60
|
|
40
|
+
|
|
41
|
+
# Ceiling for a single cooldown, including a server-supplied Retry-After
|
|
42
|
+
# (which must not be able to park a worker for an hour).
|
|
43
|
+
MAX_COOLDOWN = 900
|
|
44
|
+
|
|
45
|
+
# Consecutive throttle rounds with no intervening success before the crawl
|
|
46
|
+
# is considered banned. With the ladder above that is ~15 min of cooldown
|
|
47
|
+
# (60+120+240+480 s; the fifth round latches before its own wait).
|
|
48
|
+
GIVE_UP_AFTER = 5
|
|
49
|
+
|
|
50
|
+
# Fraction of the cooldown added as random jitter, so the workers do not
|
|
51
|
+
# all resume on the same instant and immediately re-trip the limiter.
|
|
52
|
+
JITTER_FRACTION = 0.1
|
|
53
|
+
|
|
54
|
+
# Which errors mean "you are being rate-limited", as opposed to "this
|
|
55
|
+
# resource is broken". Core ships none: the distinction is the one thing
|
|
56
|
+
# only the flavor knows, and getting it wrong in either direction is
|
|
57
|
+
# expensive — a broken resource routed here opens a pool-wide cooldown
|
|
58
|
+
# for one bad record, and a throttle routed the other way gets recorded
|
|
59
|
+
# as permanent data loss (which is exactly what the W3C crawl did before
|
|
60
|
+
# this class existed). Subclasses override the constant, or #throttle?
|
|
61
|
+
# when the signal is a status code rather than an exception class.
|
|
62
|
+
THROTTLE_ERRORS = [].freeze
|
|
63
|
+
|
|
64
|
+
# Prefix for the three duration knobs, so two flavors crawling in one
|
|
65
|
+
# process cannot share a cooldown ladder by accident. A subclass sets its
|
|
66
|
+
# own ("RELATON_W3C", "RELATON_ITU"); the suffixes are fixed.
|
|
67
|
+
ENV_PREFIX = "RELATON"
|
|
68
|
+
|
|
69
|
+
# Whether this error means the server is rate-limiting us.
|
|
70
|
+
#
|
|
71
|
+
# Class membership is the common case. A flavor whose server signals
|
|
72
|
+
# throttling with a *status code* on a shared exception class (ITU: 503 on
|
|
73
|
+
# /rec) overrides this instead of the constant.
|
|
74
|
+
#
|
|
75
|
+
# @param error [StandardError]
|
|
76
|
+
# @return [Boolean]
|
|
77
|
+
def self.throttle?(error)
|
|
78
|
+
self::THROTTLE_ERRORS.any? { |klass| error.is_a? klass }
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
attr_reader :base, :max, :give_up_after, :env_prefix
|
|
82
|
+
|
|
83
|
+
def initialize(base: nil, max: nil, give_up_after: nil,
|
|
84
|
+
clock: nil, sleeper: nil, jitter: nil, env_prefix: nil)
|
|
85
|
+
@env_prefix = env_prefix || self.class::ENV_PREFIX
|
|
86
|
+
# Remembered so #reset! can re-read the env knobs for the next crawl
|
|
87
|
+
# without discarding values a caller passed explicitly.
|
|
88
|
+
@overrides = { base: base, max: max, give_up_after: give_up_after }
|
|
89
|
+
load_limits
|
|
90
|
+
@clock = clock || -> { Time.now }
|
|
91
|
+
@sleeper = sleeper || ->(secs) { sleep secs }
|
|
92
|
+
@jitter = jitter || ->(secs) { Kernel.rand * secs * self.class::JITTER_FRACTION }
|
|
93
|
+
@mutex = Mutex.new
|
|
94
|
+
@cooldown_until = nil
|
|
95
|
+
@penalty = nil
|
|
96
|
+
@rounds = 0
|
|
97
|
+
@events = 0
|
|
98
|
+
@abandoned = false
|
|
99
|
+
@abandoned_rounds = nil
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
#
|
|
103
|
+
# Block until the shared cooldown expires. Called before every request.
|
|
104
|
+
#
|
|
105
|
+
# The mutex is never held across the sleep — the deadline is read under
|
|
106
|
+
# the lock and the sleeping happens outside it, so a cooling-down worker
|
|
107
|
+
# cannot block another thread from recording its own throttle.
|
|
108
|
+
#
|
|
109
|
+
# @return [Numeric] seconds actually slept (0.0 when there was nothing to
|
|
110
|
+
# wait for, including once the crawl has given up)
|
|
111
|
+
#
|
|
112
|
+
def wait
|
|
113
|
+
slept = 0.0
|
|
114
|
+
waited_for = nil
|
|
115
|
+
loop do
|
|
116
|
+
# Once we have given up there is nothing left to wait for, and stalling
|
|
117
|
+
# here would delay the shutdown by up to a full cooldown.
|
|
118
|
+
deadline, remaining = @mutex.synchronize do
|
|
119
|
+
gave_up? ? [nil, 0.0] : [@cooldown_until, cooldown_remaining]
|
|
120
|
+
end
|
|
121
|
+
break unless remaining.positive?
|
|
122
|
+
# Re-check after sleeping so a cooldown *extended* mid-round (a later
|
|
123
|
+
# Retry-After asking for longer) also holds back the workers already
|
|
124
|
+
# sleeping — they would otherwise wake on the stale deadline and
|
|
125
|
+
# re-trip the limiter. Looping only on a later deadline keeps a stalled
|
|
126
|
+
# clock from spinning here.
|
|
127
|
+
break if waited_for && deadline && deadline <= waited_for
|
|
128
|
+
|
|
129
|
+
waited_for = deadline
|
|
130
|
+
delay = remaining + @jitter.call(remaining)
|
|
131
|
+
@sleeper.call(delay)
|
|
132
|
+
slept += delay
|
|
133
|
+
end
|
|
134
|
+
slept
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
#
|
|
138
|
+
# Record a 429 and open/extend the shared cooldown.
|
|
139
|
+
#
|
|
140
|
+
# @param retry_after [Integer, nil] seconds from the response's Retry-After
|
|
141
|
+
#
|
|
142
|
+
# @return [Boolean] true when the crawl should give up
|
|
143
|
+
#
|
|
144
|
+
def throttled!(retry_after: nil)
|
|
145
|
+
@mutex.synchronize do
|
|
146
|
+
now = @clock.call
|
|
147
|
+
@events += 1
|
|
148
|
+
if @cooldown_until.nil? || now >= @cooldown_until
|
|
149
|
+
open_round(now, retry_after)
|
|
150
|
+
elsif retry_after
|
|
151
|
+
# Mid-round, the server may still ask for longer than we planned.
|
|
152
|
+
extend_round(now + capped(retry_after))
|
|
153
|
+
end
|
|
154
|
+
# Giving up latches. Otherwise a straggler worker succeeding after the
|
|
155
|
+
# producer already stopped paginating would clear @rounds and let the
|
|
156
|
+
# crawl save the truncated index it just decided to abandon.
|
|
157
|
+
#
|
|
158
|
+
# @rounds is snapshotted at the same moment, because #succeeded!
|
|
159
|
+
# resets it: read afterwards it names whatever has re-accumulated
|
|
160
|
+
# since, not the threshold that was crossed. The abort message is
|
|
161
|
+
# the only place an operator learns why the crawl stopped.
|
|
162
|
+
unless @abandoned
|
|
163
|
+
@abandoned = @rounds >= @give_up_after
|
|
164
|
+
@abandoned_rounds = @rounds if @abandoned
|
|
165
|
+
end
|
|
166
|
+
@abandoned
|
|
167
|
+
end
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
#
|
|
171
|
+
# Record a successful request: the limiter has released us.
|
|
172
|
+
#
|
|
173
|
+
def succeeded!
|
|
174
|
+
@mutex.synchronize do
|
|
175
|
+
@rounds = 0
|
|
176
|
+
@cooldown_until = nil
|
|
177
|
+
# Decay rather than reset, so a limiter that keeps flapping still
|
|
178
|
+
# climbs the ladder instead of restarting at `base` every time.
|
|
179
|
+
@penalty = @penalty && @penalty > @base ? [@penalty / 2.0, @base].max : nil
|
|
180
|
+
end
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
# @return [Boolean] give_up_after consecutive rounds passed with no success
|
|
184
|
+
def exhausted?
|
|
185
|
+
@mutex.synchronize { gave_up? }
|
|
186
|
+
end
|
|
187
|
+
|
|
188
|
+
# Clear all state for a fresh crawl. Resets in place rather than handing
|
|
189
|
+
# back a new object so callers (and specs) holding this instance keep
|
|
190
|
+
# observing the live governor.
|
|
191
|
+
def reset!
|
|
192
|
+
@mutex.synchronize do
|
|
193
|
+
load_limits
|
|
194
|
+
@cooldown_until = nil
|
|
195
|
+
@penalty = nil
|
|
196
|
+
@rounds = 0
|
|
197
|
+
@events = 0
|
|
198
|
+
@abandoned = false
|
|
199
|
+
@abandoned_rounds = nil
|
|
200
|
+
end
|
|
201
|
+
end
|
|
202
|
+
|
|
203
|
+
# @return [Integer] consecutive throttle rounds since the last success
|
|
204
|
+
def throttle_rounds
|
|
205
|
+
@mutex.synchronize { @rounds }
|
|
206
|
+
end
|
|
207
|
+
|
|
208
|
+
# The consecutive-round count as it stood when the give-up latched, for
|
|
209
|
+
# the abort message. Distinct from #throttle_rounds, which stays LIVE
|
|
210
|
+
# because a crawler logs it per retry as the current round (see
|
|
211
|
+
# Relaton::Itu::DataCrawlerR) and #succeeded! resets it.
|
|
212
|
+
#
|
|
213
|
+
# @return [Integer, nil] nil while the crawl is still running
|
|
214
|
+
def give_up_rounds
|
|
215
|
+
@mutex.synchronize { @abandoned_rounds }
|
|
216
|
+
end
|
|
217
|
+
|
|
218
|
+
# @return [Integer] every 429 observed, for reporting
|
|
219
|
+
def throttle_count
|
|
220
|
+
@mutex.synchronize { @events }
|
|
221
|
+
end
|
|
222
|
+
|
|
223
|
+
#
|
|
224
|
+
# Seconds requested by a rate-limited response's Retry-After header.
|
|
225
|
+
#
|
|
226
|
+
# lutaml-hal >= 0.2.5 carries a `{ status:, headers: }` hash on every error
|
|
227
|
+
# mapped from a response (nil for locally raised ones), so anything without
|
|
228
|
+
# one yields nil.
|
|
229
|
+
#
|
|
230
|
+
# @param error [StandardError]
|
|
231
|
+
#
|
|
232
|
+
# @return [Integer, nil]
|
|
233
|
+
#
|
|
234
|
+
def self.retry_after(error)
|
|
235
|
+
return nil unless error.respond_to?(:response)
|
|
236
|
+
|
|
237
|
+
response = error.response
|
|
238
|
+
headers = response.is_a?(Hash) ? (response[:headers] || response["headers"]) : nil
|
|
239
|
+
return nil unless headers.respond_to?(:[])
|
|
240
|
+
|
|
241
|
+
seconds_from headers["retry-after"] || headers["Retry-After"]
|
|
242
|
+
end
|
|
243
|
+
|
|
244
|
+
#
|
|
245
|
+
# Retry-After is either delta-seconds or an HTTP-date (RFC 9110). The date
|
|
246
|
+
# form is parsed properly rather than by scanning digits, which would pick
|
|
247
|
+
# up the day-of-month and back off for a wildly wrong duration; a date
|
|
248
|
+
# already in the past yields 0, i.e. "no extra floor".
|
|
249
|
+
#
|
|
250
|
+
def self.seconds_from(value)
|
|
251
|
+
str = value.to_s.strip
|
|
252
|
+
return nil if str.empty?
|
|
253
|
+
return str.to_i if str.match?(/\A\d+\z/)
|
|
254
|
+
|
|
255
|
+
begin
|
|
256
|
+
[(Time.parse(str) - Time.now).ceil, 0].max
|
|
257
|
+
rescue ArgumentError
|
|
258
|
+
nil
|
|
259
|
+
end
|
|
260
|
+
end
|
|
261
|
+
|
|
262
|
+
# A blank or non-positive override is treated as unset.
|
|
263
|
+
def self.env_int(name, default)
|
|
264
|
+
value = ENV[name].to_s.strip
|
|
265
|
+
return default if value.empty?
|
|
266
|
+
|
|
267
|
+
parsed = value.to_i
|
|
268
|
+
parsed.positive? ? parsed : default
|
|
269
|
+
end
|
|
270
|
+
|
|
271
|
+
private
|
|
272
|
+
|
|
273
|
+
# "<PREFIX>_THROTTLE_<SUFFIX>", the one thing ENV_PREFIX exists for.
|
|
274
|
+
def env_name(suffix)
|
|
275
|
+
"#{@env_prefix}_THROTTLE_#{suffix}"
|
|
276
|
+
end
|
|
277
|
+
|
|
278
|
+
# Read the duration knobs, letting explicit constructor args win over the
|
|
279
|
+
# environment. Re-read on #reset! so the env vars behave like a flavor's
|
|
280
|
+
# sibling knobs, which are class methods evaluated per crawl rather than
|
|
281
|
+
# frozen when the flavor was autoloaded.
|
|
282
|
+
def load_limits
|
|
283
|
+
@base = @overrides[:base] || self.class.env_int(env_name("BASE"), self.class::BASE_COOLDOWN)
|
|
284
|
+
@max = @overrides[:max] || self.class.env_int(env_name("MAX"), self.class::MAX_COOLDOWN)
|
|
285
|
+
@give_up_after = @overrides[:give_up_after] ||
|
|
286
|
+
self.class.env_int(env_name("GIVEUP"), self.class::GIVE_UP_AFTER)
|
|
287
|
+
end
|
|
288
|
+
|
|
289
|
+
# Start a new escalation step. Retry-After raises the floor but never
|
|
290
|
+
# lowers our own ladder — the server's hint is a minimum, not a discount.
|
|
291
|
+
def open_round(now, retry_after)
|
|
292
|
+
@rounds += 1
|
|
293
|
+
ladder = @penalty ? [@penalty * 2, @max].min : @base
|
|
294
|
+
hint = retry_after ? capped(retry_after) : 0
|
|
295
|
+
@penalty = [ladder, hint].max
|
|
296
|
+
@cooldown_until = now + @penalty
|
|
297
|
+
end
|
|
298
|
+
|
|
299
|
+
def extend_round(deadline)
|
|
300
|
+
@cooldown_until = deadline if deadline > @cooldown_until
|
|
301
|
+
end
|
|
302
|
+
|
|
303
|
+
def cooldown_remaining
|
|
304
|
+
return 0.0 unless @cooldown_until
|
|
305
|
+
|
|
306
|
+
[@cooldown_until - @clock.call, 0.0].max
|
|
307
|
+
end
|
|
308
|
+
|
|
309
|
+
def capped(seconds)
|
|
310
|
+
[seconds, @max].min
|
|
311
|
+
end
|
|
312
|
+
|
|
313
|
+
# Callers must already hold the mutex. Latched by #throttled! — never
|
|
314
|
+
# recomputed from @rounds, which #succeeded! resets.
|
|
315
|
+
def gave_up?
|
|
316
|
+
@abandoned
|
|
317
|
+
end
|
|
318
|
+
end
|
|
319
|
+
end
|
|
320
|
+
end
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
module Relaton
|
|
2
|
+
module Core
|
|
3
|
+
module HashKeysSymbolizer
|
|
4
|
+
def symbolize_hash_keys(obj)
|
|
5
|
+
case obj
|
|
6
|
+
when Array
|
|
7
|
+
obj.map { |e| symbolize_hash_keys(e) }
|
|
8
|
+
when Hash
|
|
9
|
+
obj.each_with_object({}) do |(k, v), h|
|
|
10
|
+
key = k.is_a?(String) ? k.to_sym : k
|
|
11
|
+
h[key] = symbolize_hash_keys(v)
|
|
12
|
+
end
|
|
13
|
+
else
|
|
14
|
+
obj
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
end
|
|
19
|
+
end
|