anystyle 1.3.13 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '09ec60a796bf13a493c0e91f4df8ad0295d70dde460da704119dfcbbd654a09c'
-  data.tar.gz: e17787d8d6a2007b9f815e64c32092f305fdf92e80ec3914da1a72318ec049ee
+  metadata.gz: b7d5642b2c453c00a91748d6e447504c825b0097095cda159d92f2c6d82a11c5
+  data.tar.gz: c29fa4ee58a017404b205418bcf1d45f7111d3005e881f5d869d73572d0cd5e6
 SHA512:
-  metadata.gz: ab5a9c7bfeb5b65c0b6345d69a11bcae62ea8df785f3eb212c1e8d345e2882ac44da53098523cc3e07828ca0b0da8557de133e1e95ac21f35fe3f5d9ca405079
-  data.tar.gz: 408f6b7f3810e31984c3e8dbbca1831bf98a1197b443d233ff831b824af5bd8d4249935eaa4e3fc02bc0b8d15e7c4d85256e82253c4c8ec60500b8c269d70ff3
+  metadata.gz: 44f8226f33b22aa53b743ce56db3203a1c1bb3e4dd749ca746a0c46563656f36a4da585eebb32f0b8062701f6719e5c0aed9ee5f9fa533a1f49aacdef2d79504
+  data.tar.gz: dcaf21ed636bc50a7fa982e220942b0d752f13714e1a8cda038d1d7b324b7da9c78569903460c10454c5740d191e1abc19254c24222cd2a411880c78fe3933ea

data/HISTORY.md

@@ -1,3 +1,9 @@
+1.4.0 / 2023-01-06
+==================
+* Removed deprectate string taint checking (@bbonamin).
+* `AnyStyle::Parser#parse` will no longer automatically open local files.
+  Please call `Wapiti::Dataset.open` explicitly if you relied on this.
+
 1.3.6 / 2019-12-02
 ==================
 * Updated parser model.

data/LICENSE

@@ -1,5 +1,5 @@
 AnyStyle
-Copyright 2011-2020 Sylvester Keil. All rights reserved.
+Copyright 2011-2023 Sylvester Keil. All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions are met:

data/README.md

@@ -57,6 +57,9 @@ AnyStyle is available as web application at [anystyle.io](https://anystyle.io).
 The web application [is open source](https://github.com/inukshuk/anystyle.io)
 and you can also host yourself!
 
+Improving results for your data
+=================================
+
 Training
 --------
 You can train custom Finder and Parser models. To do this, you need
@@ -105,6 +108,47 @@ When working with training data, it is a good idea to use the
 `Wapiti::Dataset` API in Ruby: it supports all the standard set
 operators and makes it very easy to combine or compare data sets.
 
+Natural Languages used in AnyStyle
+----------------------------------
+
+As mentioned above, the
+[core](https://github.com/inukshuk/anystyle/blob/master/res/parser/core.xml)
+dataset contains the manually marked-up references that are used as the
+basis for the default AnyStyle parsing model. If the references you are
+trying to parse include many non-English documents, the distribution of
+natural languages in this corpus is relevant (detected using [cld](https://github.com/jtoy/cld)).
+
+| Language                | n   |
+|-------------------------|-----|
+| ENGLISH                 | 965 |
+| FRENCH                  | 54  |
+| GERMAN                  | 26  |
+| ITALIAN                 | 11  |
+| Others                  | 9   |
+|                         |     |
+| Not reliably determined | 449 |
+| (but mainly English)    |     |
+
+(These data are based on AnyStyle version 1.3.13) 
+
+There is a strong prevalence of English-language documents with the
+conventions used in English-language bibliographies, with some
+representation of other European languages. The languages used reflect
+those used in scientific publishing as well as the maintainers'
+competencies. If you are working with many documents in languages other
+than English, you might consider training the model with some examples
+in the relevant languages.
+
+AnyStyle should work with references written in any Latin script
+(including most European languages, languages such as Indonesian and
+Malaysian, as well as romanised Arabic, Chinese and Japanese). It should
+also support languages written with non-Latin alphabets (such as
+Russian), although no examples of these appear in the default training
+sets. Languages written in syllabaries or complex symbols which do not
+use white space to separate tokens are not compatible with AnyStyle's
+approach: this includes Chinese, Japanese, Arabic as well as many Indian
+languages. 
+
 Dictionary Adapters
 -------------------
 During the statistical analysis of reference strings, AnyStyle relies
@@ -142,6 +186,8 @@ and configure AnyStyle to use the Redis adapter:
     AnyStyle::Dictionary::Redis.defaults[:host] = 'localhost'
     AnyStyle::Dictionary::Redis.defaults[:port] = 6379
 
+About AnyStyle
+==============
 Contributing
 ------------
 The AnyStyle source code is
@@ -167,7 +213,7 @@ to join us! Over the years our main contributors have been:
 
 License
 -------
-Copyright 2011-2020 Sylvester Keil. All rights reserved.
+Copyright 2011-2023 Sylvester Keil. All rights reserved.
 
 AnyStyle is distributed under a BSD-style license.
 See LICENSE for details.

data/lib/anystyle/dictionary/marshal.rb

@@ -10,7 +10,7 @@ module AnyStyle
       end
 
       def open
-        if File.exists?(options[:path])
+        if File.exist?(options[:path])
           @db = ::Marshal.load(File.open(options[:path]))
         else
           @db = {}

data/lib/anystyle/document.rb

@@ -18,8 +18,6 @@ module AnyStyle
       end
 
       def open(path, format: File.extname(path), tagged: false, **opts)
-        raise ArgumentError,
-          "cannot open tainted path: '#{path}'" if path.tainted?
         raise ArgumentError,
           "document not found: '#{path}'" unless File.exist?(path)
 

data/lib/anystyle/finder.rb

@@ -8,7 +8,7 @@ module AnyStyle
       compact: true,
       threads: 4,
       format: :references,
-      training_data: Dir[File.join(RES, 'finder', '*.ttx')].map(&:untaint),
+      training_data: Dir[File.join(RES, 'finder', '*.ttx')],
       layout: true,
       pdftotext: 'pdftotext',
       pdfinfo: 'pdfinfo'

data/lib/anystyle/normalizer/locator.rb

@@ -13,7 +13,12 @@ module AnyStyle
           when :url
             doi = doi_extract(value) if value =~ /doi\.org\//i
             append item, :doi, doi unless doi.nil?
-            URI.extract(value)
+            urls = URI.extract(value, %w(http https ftp ftps))
+            if urls.empty?
+              value
+            else
+              urls
+            end
           when :doi
             doi_extract(value) || value
           else

data/lib/anystyle/parser.rb

@@ -18,7 +18,8 @@ module AnyStyle
     attr_reader :model, :options, :features, :normalizers, :mtime
 
     def initialize(options = {})
-      @options = self.class.defaults.merge(options)
+      def_opts = self.class.defaults || Parser.defaults
+      @options = def_opts.merge(options)
       load_model
     end
 
@@ -85,12 +86,6 @@ module AnyStyle
         expand input
       when Wapiti::Sequence
         expand Wapiti::Dataset.new([input])
-      when String
-        if !input.tainted? && input.length < 1024 && File.exists?(input)
-          expand Wapiti::Dataset.open(input, **opts)
-        else
-          expand Wapiti::Dataset.parse(input, **opts)
-        end
       else
         expand Wapiti::Dataset.parse(input, **opts)
       end