anystyle 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 694288a5b219077818965fffd8eed72aa01c3a096a99647c2ee6014edeb13e38
+  data.tar.gz: db7aefb69e10d359aaf55a1da8d0c6041bfefb91088f5e1cc330e6bb3a13e005
+SHA512:
+  metadata.gz: 595b8627b93a7e61912fc65f1d1c14a4a0dbd6263d7612c21184aa1e9e42856e16c265a29c6e8b2b3588da1b6a70ca456145ae72c4f4980fea502a7fde314de7
+  data.tar.gz: d14ae39fd0e156e76619b8b23e171a8b364fa1d14e00c4f9517edd8bc40d5ff082c83073343fcb345f42a3a5b6828c4358a29593067f35696584d8e868113cfa

data/HISTORY.md

@@ -0,0 +1,78 @@
+0.6.0 / 2014-03-24
+==================
+* Renamed tech label to genre; institution to authority
+* Updated classifier
+* Updated model
+
+0.5.3 / 2014-03-14
+==================
+* Added source, director, producer and section labels
+* Updated model
+
+0.5.2 / 2014-03-13
+==================
+* Add XML output
+* Improve ULR normalizer
+
+0.4.4 & 0.4.5 / 2014-03-10
+==========================
+* Mitigate potential vulnerability: open files only if string not tainted
+
+0.4.3 / 2014-03-09
+==================
+* Add model reload method
+
+0.4.2 / 2014-03-08
+==================
+* Add redis-namespace support
+
+0.4.1 / 2014-03-03
+==================
+* Normalizer tweaks
+
+0.4.0 / 2014-02-27
+==================
+* Update wapiti
+* Improve dash patterns
+* Updated default model
+
+0.3.0 / 2014-02-14
+==================
+* Update dependencies
+* Added raw output format
+* Enforce Ruby 1.9.3 or later requirement
+
+0.2.0 / 2012-10-29
+==================
+* Use Namae for name normalization
+
+0.1.1 / 2012-03-29
+==================
+* Bugfix
+
+0.1.0 / 2012-03-03
+==================
+* Added redis as data store option
+
+0.0.10 / 2012-03-01
+===================
+* Added new output format: tags (to generate training data)
+
+0.0.9 / 2011-09-08
+==================
+* Added year-range / page-range feature distinction
+* Improved model and training data
+* Added URL, DOI and ISBN recognition features
+* Improved handling of UTF-8 characters in Regular Expressions
+* Bugfixes
+
+0.0.6 / 2011-09-06
+==================
+* Added location normalizer
+* Improved punctuation feature elicitation
+* Improved name tokenizing
+* Bugfixes
+
+0.0.1 / 2011-09-05
+==================
+* Initial release

data/LICENSE

@@ -0,0 +1,27 @@
+AnyStyle
+Copyright 2011-2018 Sylvester Keil. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+ 1. Redistributions of source code must retain the above copyright notice,
+    this list of conditions and the following disclaimer.
+
+ 2. Redistributions in binary form must reproduce the above copyright notice,
+    this list of conditions and the following disclaimer in the documentation
+    and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER ``AS IS'' AND ANY EXPRESS OR
+IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+The views and conclusions contained in the software and documentation are
+those of the authors and should not be interpreted as representing official
+policies, either expressed or implied, of the copyright holder.

data/README.md

@@ -0,0 +1,103 @@
+AnyStyle
+========
+[![Build Status](https://travis-ci.org/inukshuk/anystyle.svg?branch=master)](https://travis-ci.org/inukshuk/anystyle)
+[![Coverage Status](https://coveralls.io/repos/github/inukshuk/anystyle/badge.svg?branch=master)](https://coveralls.io/github/inukshuk/anystyle?branch=master)
+
+AnyStyle is a very fast and smart parser for academic references. It
+was originally inspired by [ParsCit](http://aye.comp.nus.edu.sg/parsCit/)
+and [FreeCite](http://freecite.library.brown.edu/); AnyStyle uses machine
+learning algorithms and aims to make it easy to train the model with data
+that is relevant to your parsing needs.
+
+
+Using AnyStyle CLI
+------------------
+
+    $ [sudo] gem install anystyle-cli
+    $ anystyle --help
+    $ anystyle help find
+    $ anystyle help parse
+
+See [anystyle-cli](https://github.com/inukshuk/anystyle-cli) for more details.
+
+
+Web Application and Web Service
+-------------------------------
+AnyStyle is available as a web-application and service at
+[anystyle.io](https://anystyle.io).
+
+Please note that the web service is currently based on the legacy
+[0.x branch](https://github.com/inukshuk/anystyle/tree/0.x).
+
+
+Using AnyStyle in Ruby
+----------------------
+
+    $ [sudo] gem install anystyle
+
+
+Reference Parsing
+-----------------
+
+Document Parsing
+----------------
+
+Training
+--------
+
+Dictionary Adapters
+-------------------
+During the statistical analysis of reference strings, AnyStyle relies
+on a large feature dictionary; by default, AnyStyle creates a persistent
+Ruby Hash in the folder of the `anystyle-data` Gem. This uses up about
+2MB of disk space and keeps the entire dictionary in memory. If you prefer
+a smaller memory footprint, you can alternatively use AnyStyle's GDBM
+dictionary. GDBM bindings are part of the Ruby standard library and are
+supported on all platforms, but you may have to install GDBM on your
+platform before installing Ruby.
+
+If you do not want to use the the persistent Ruyb Hash nor the GBDM
+bindings, you can store your dictionary in memory (not recommended) or
+use a Redis. The best way to change the default dictionary adapter is by
+adjusting AnyStyle's default configuration (when using the default parser
+instances you must set the default before using the parser):
+
+    AnyStyle::Dictionary.defaults[:adapter] = :ruby
+    #-> Use a persistent Ruby hash;
+    #-> slower start-up than GDBM but no extra dependency
+
+    AnyStyle::Dictionary.defaults[:adapter] = :hash
+    #-> Use in-memory dictionary; slow start-up but uses no space on disk
+
+    require 'anystyle/dictionary/gdbm'
+    AnyStyle::Dictionary.defaults[:adapter] = :gdbm
+
+To use Redis, install the `redis` and `redis/namespace` (optional) Gems
+and configure AnyStyle to use the Redis adapter:
+
+    AnyStyle::Dictionary.defaults[:adapter] = :redis
+
+    # Adjust the Redis-specifi configuration
+    require 'anystyle/dictionary/redis'
+    AnyStyle::Dictionary::Redis.defaults[:host] = 'localhost'
+    AnyStyle::Dictionary::Redis.defaults[:port] = 6379
+
+Contributing
+------------
+The AnyStyle source code is
+[hosted on GitHub](https://github.com/inukshuk/anystyle/).
+You can check out a copy of the latest code using Git:
+
+    $ git clone https://github.com/inukshuk/anystyle.git
+
+If you've found a bug or have a question, please open an issue on the
+[AnyStyle issue tracker](http://github.com/inukshuk/anystyle/issues).
+Or, for extra credit, clone the AnyStyle repository, write a failing
+example, fix the bug and submit a pull request.
+
+License
+-------
+Copyright 2011-2018 Sylvester Keil. All rights reserved.
+
+AnyStyle is distributed under a BSD-style license.
+See LICENSE for details.

data/lib/anystyle.rb

@@ -0,0 +1,71 @@
+require 'forwardable'
+require 'wapiti'
+
+require 'anystyle/version'
+require 'anystyle/support'
+require 'anystyle/errors'
+require 'anystyle/utils'
+require 'anystyle/dictionary'
+require 'anystyle/dictionary/marshal'
+require 'anystyle/data'
+
+require 'anystyle/feature'
+require 'anystyle/feature/affix'
+require 'anystyle/feature/brackets'
+require 'anystyle/feature/canonical'
+require 'anystyle/feature/caps'
+require 'anystyle/feature/category'
+require 'anystyle/feature/dictionary'
+require 'anystyle/feature/indent'
+require 'anystyle/feature/keyword'
+require 'anystyle/feature/line'
+require 'anystyle/feature/locator'
+require 'anystyle/feature/number'
+require 'anystyle/feature/position'
+require 'anystyle/feature/punctuation'
+require 'anystyle/feature/ref'
+require 'anystyle/feature/terminal'
+require 'anystyle/feature/words'
+
+require 'anystyle/normalizer'
+require 'anystyle/normalizer/brackets'
+require 'anystyle/normalizer/container'
+require 'anystyle/normalizer/date'
+require 'anystyle/normalizer/edition'
+require 'anystyle/normalizer/journal'
+require 'anystyle/normalizer/locale'
+require 'anystyle/normalizer/location'
+require 'anystyle/normalizer/locator'
+require 'anystyle/normalizer/names'
+require 'anystyle/normalizer/page'
+require 'anystyle/normalizer/publisher'
+require 'anystyle/normalizer/pubmed'
+require 'anystyle/normalizer/punctuation'
+require 'anystyle/normalizer/quotes'
+require 'anystyle/normalizer/type'
+require 'anystyle/normalizer/volume'
+
+require 'anystyle/format/bibtex'
+require 'anystyle/format/csl'
+
+require 'anystyle/document'
+require 'anystyle/parser'
+require 'anystyle/finder'
+
+module AnyStyle
+  def self.parser
+    Parser.instance
+  end
+
+  def self.parse(*arguments)
+    parser.parse(*arguments)
+  end
+
+  def self.finder
+    Finder.instance
+  end
+
+  def self.find(*arguments)
+    finder.find(*arguments)
+  end
+end

data/lib/anystyle/dictionary.rb

@@ -0,0 +1,132 @@
+module AnyStyle
+  class Dictionary
+    @tags = [:name, :place, :publisher, :journal]
+
+    @code = Hash[
+      *@tags.zip(0.upto(@tags.length-1).map { |i| 2**i }).flatten
+    ]
+
+    @tags.freeze
+    @code.freeze
+
+    @defaults = {
+      adapter: :ruby,
+      source: nil
+    }
+
+    class << self
+      attr_reader :tags, :code, :defaults, :adapters
+
+      def create(options = {})
+        return options if options.is_a?(Dictionary)
+
+        options = defaults.merge(options || {})
+        adapter = options.delete :adapter
+
+        case adapter.to_sym
+        when :memory, :hash
+          new options
+        when :gdbm
+          require 'anystyle/dictionary/gdbm'
+          Dictionary::GDBM.new options
+        when :lmdb
+          require 'anystyle/dictionary/lmdb'
+          Dictionary::LMDB.new options
+        when :redis
+          require 'anystyle/dictionary/redis'
+          Dictionary::Redis.new options
+        when :marshal, :ruby
+          require 'anystyle/dictionary/marshal'
+          Dictionary::Marshal.new options
+        else
+          raise ArgumentError, "unknown adapter: #{adapter}"
+        end
+      end
+
+      def instance
+        Thread.current['anystyle_dictionary'] ||= create.open
+      end
+    end
+
+    attr_reader :db, :options
+
+    def initialize(options)
+      @options = options
+    end
+
+    def open
+      @db = {} unless open?
+      self
+    ensure
+      populate! if empty?
+    end
+
+    def close
+      @db = nil
+    end
+
+    def truncate
+      close
+    end
+
+    def open?
+      not db.nil?
+    end
+
+    def empty?
+      db.empty?
+    end
+
+    def get(key)
+      db[key.to_s].to_i
+    end
+
+    def put(key, value)
+      db[key.to_s] = value.to_i
+    end
+
+    alias_method :[], :get
+    alias_method :[]=, :put
+
+    def tags(key)
+      value = get key
+
+      Dictionary.tags.map { |tag|
+        (value & Dictionary.code[tag] > 0) ? 'T' : 'F'
+      }
+    end
+
+    def tag_counts(keys)
+      counts = Dictionary.tags.map { 0 }
+      keys.each do |key|
+        value = get(key)
+        Dictionary.tags.each.with_index do |tag, idx|
+          counts[idx] += 1 if (value & Dictionary.code[tag] > 0)
+        end if value > 0
+      end
+      counts
+    end
+
+    def populate!
+      require 'zlib'
+
+      File.open(options[:source], 'rb') do |file|
+        mode = 0
+
+        Zlib::GzipReader.new(file, encoding: 'UTF-8').each do |line|
+          line.strip!
+
+          case line
+          when /^#! (\w+)/i
+            mode = Dictionary.code[$1.to_sym]
+          when /^#/
+            # skip comments
+          else
+            key = line.split(/\s+(\d+\.\d+)\s*$/)[0]
+            put key, get(key) | mode
+          end
+        end
+      end
+    end
+  end
+end

data/lib/anystyle/dictionary/gdbm.rb

@@ -0,0 +1,52 @@
+module AnyStyle
+  require 'gdbm'
+
+  class Dictionary
+    class GDBM < Dictionary
+      @defaults = {
+        path: File.expand_path('../../data/dict.db', __FILE__),
+        mode: 0666,
+        flags: ::GDBM::WRCREAT | ::GDBM::NOLOCK
+      }
+
+      attr_reader :env
+
+      def initialize(options = {})
+        super(self.class.defaults.merge(options))
+      end
+
+      def open
+        close
+        @db = ::GDBM.new(*options.values_at(:path, :mode, :flags))
+        self
+      ensure
+        populate! if empty?
+      end
+
+      def close
+        db.close if open?
+      end
+
+      def open?
+        !(db.nil? || db.closed?)
+      end
+
+      def empty?
+        open? and db.empty?
+      end
+
+      def truncate
+        close
+        File.unlink(options[:path])
+      end
+
+      def get(key)
+        db[key.to_s].to_i
+      end
+
+      def put(key, value)
+        db[key.to_s] = value.to_i.to_s
+      end
+    end
+  end
+end