anystyle 1.3.11 → 1.3.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dcb19c0b21f5cd2ba5409b22ff1105e5186981871c22cf6862633cdaa0150a61
-  data.tar.gz: b0fcd484f3f87784816bd4e3dc55e059329602f4800242362531cf25c9db7adc
+  metadata.gz: 1bf14266569ad0eb5e812e68d0289780e50ee2cc765faeb1a5fb48b37b1ba4b7
+  data.tar.gz: be96508a8b939c6450342a763cfb469b87d170dab80380b4a82beb7d02b4e0c4
 SHA512:
-  metadata.gz: f0df92b83ca46a7464c94c737dd035914d2d67c3269a7ca1730225511f0a57e4fe3530f1cedf8d1a63894650946bb953c178de9f2fd584411d72aa5f62b31290
-  data.tar.gz: 975dddfcf906495bc0c65b26d816e372bfc9fbf0331f1b08350cb9fc0befcff3754968f620501682f43f0985885d8e4319017458ba28b4092ab34038fe7043f0
+  metadata.gz: a8a2c6f4f0d997b657b9817b8bf2a4bdf62db4e9f53130ed9923de571e9cb6a264e554dd78c4ebaf7180b424734632d526fd46d3accba07e0cd766fb37689fae
+  data.tar.gz: d2ddd8f136ac10a0ddd29b9522fc7779c1ea92c1d441601c801726fa4e304d104de588b38c230fa53102e59e23e2ce0b48b79f4c4bd932c2cb0e3a3f56ccd43a

data/README.md

@@ -57,6 +57,9 @@ AnyStyle is available as web application at [anystyle.io](https://anystyle.io).
 The web application [is open source](https://github.com/inukshuk/anystyle.io)
 and you can also host yourself!
 
+Improving results for your data
+=================================
+
 Training
 --------
 You can train custom Finder and Parser models. To do this, you need
@@ -105,6 +108,47 @@ When working with training data, it is a good idea to use the
 `Wapiti::Dataset` API in Ruby: it supports all the standard set
 operators and makes it very easy to combine or compare data sets.
 
+Natural Languages used in AnyStyle
+----------------------------------
+
+As mentioned above, the
+[core](https://github.com/inukshuk/anystyle/blob/master/res/parser/core.xml)
+dataset contains the manually marked-up references that are used as the
+basis for the default AnyStyle parsing model. If the references you are
+trying to parse include many non-English documents, the distribution of
+natural languages in this corpus is relevant (detected using [cld](https://github.com/jtoy/cld)).
+
+| Language                | n   |
+|-------------------------|-----|
+| ENGLISH                 | 965 |
+| FRENCH                  | 54  |
+| GERMAN                  | 26  |
+| ITALIAN                 | 11  |
+| Others                  | 9   |
+|                         |     |
+| Not reliably determined | 449 |
+| (but mainly English)    |     |
+
+(These data are based on AnyStyle version 1.3.13) 
+
+There is a strong prevalence of English-language documents with the
+conventions used in English-language bibliographies, with some
+representation of other European languages. The languages used reflect
+those used in scientific publishing as well as the maintainers'
+competencies. If you are working with many documents in languages other
+than English, you might consider training the model with some examples
+in the relevant languages.
+
+AnyStyle should work with references written in any Latin script
+(including most European languages, languages such as Indonesian and
+Malaysian, as well as romanised Arabic, Chinese and Japanese). It should
+also support languages written with non-Latin alphabets (such as
+Russian), although no examples of these appear in the default training
+sets. Languages written in syllabaries or complex symbols which do not
+use white space to separate tokens are not compatible with AnyStyle's
+approach: this includes Chinese, Japanese, Arabic as well as many Indian
+languages. 
+
 Dictionary Adapters
 -------------------
 During the statistical analysis of reference strings, AnyStyle relies
@@ -142,6 +186,8 @@ and configure AnyStyle to use the Redis adapter:
     AnyStyle::Dictionary::Redis.defaults[:host] = 'localhost'
     AnyStyle::Dictionary::Redis.defaults[:port] = 6379
 
+About AnyStyle
+==============
 Contributing
 ------------
 The AnyStyle source code is

data/lib/anystyle/normalizer/locator.rb

@@ -13,7 +13,12 @@ module AnyStyle
           when :url
             doi = doi_extract(value) if value =~ /doi\.org\//i
             append item, :doi, doi unless doi.nil?
-            URI.extract(value)
+            urls = URI.extract(value, %w(http https ftp ftps))
+            if urls.empty?
+              value
+            else
+              urls
+            end
           when :doi
             doi_extract(value) || value
           else

data/lib/anystyle/normalizer/volume.rb

@@ -4,6 +4,7 @@ module AnyStyle
       @keys = [:volume, :pages, :date]
 
       VOLNUM_RX = '(\p{Lu}?\d+|[IVXLCDM]+)'
+
       def normalize(item, **opts)
         map_values(item, [:volume]) do |_, volume|
           volume = StringUtils.strip_html volume
@@ -15,7 +16,7 @@ module AnyStyle
           end
 
           case volume
-          when /(?:^|\s)#{VOLNUM_RX}\s?\(([^)]+)\)(\s?\d+\p{Pd}\d+)?/
+          when /(?:^|\s)#{VOLNUM_RX}\s?\(([^)]+)\)[;:,]?(?:pp?.?)?(\s?\d+\p{Pd}\d+)?/
             volume = $1
             append item, :issue, $2
             append item, :pages, $3.strip unless $3.nil?

data/lib/anystyle/parser.rb

@@ -6,7 +6,7 @@ module AnyStyle
       attr_reader :defaults, :formats
 
       def load(path)
-        new :model => path
+        new model: path
       end
 
       # Returns a default parser instance
@@ -18,7 +18,8 @@ module AnyStyle
     attr_reader :model, :options, :features, :normalizers, :mtime
 
     def initialize(options = {})
-      @options = self.class.defaults.merge(options)
+      def_opts = self.class.defaults || Parser.defaults
+      @options = def_opts.merge(options)
       load_model
     end
 
@@ -28,7 +29,7 @@ module AnyStyle
         @model.options.update_attributes options
         @mtime = File.mtime(file)
       else
-        @model = Wapiti::Model.new(options.reject { |k,_| k == :model })
+        @model = Wapiti::Model.new(options.reject { |k, _| k == :model })
         @model.path = options[:model]
         @mtime = Time.now
       end