bel_parser 1.0.0.alpha.16 → 1.0.0.alpha.17

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3e827bc3f99b0c849e302bd2f32debd794abf423
-  data.tar.gz: e18d99921f436e6eb6d684230252d3c213d470da
+  metadata.gz: 21d0de137c850359607608e5cdca2a4481934fd5
+  data.tar.gz: aebe4a51435a51ee91d410a2c5ff0753ec3d51d2
 SHA512:
-  metadata.gz: 6c25495a94d389796b6a6f086527fc1c0ce6d6a6850966eaa169a6961d970f853ff03ec201022fd86609011ed0a3dfd76670e88ef4afdd18c749b43a02aa56ab
-  data.tar.gz: a3fbb92d0168fe0d7373ba01cc9a986f8127be586aefe31d3a93117ff72506ea406daa073979fbbaa5af5ae98360858ed1edf1289550a485c950fb42736894a4
+  metadata.gz: 51d92527013676307ee3307496f1282fe838a9f57b138e1d74d6dcdb49cd7061fd913c26111422f9e1fcad3c61dcbd1ead176996ec90e20acc2e272b49de710b
+  data.tar.gz: edf6a891d9d6d9ba08dcef2cf4f6090cba96b040b931c8ee7aa39296bc7a6a305b81b45c2e617254b9b293a92a0e90db80756327b2b6e90430b1c8b68caeb184

data/VERSION

@@ -1 +1 @@
-1.0.0.alpha.16
+1.0.0.alpha.17

data/bin/bel2_validator

@@ -9,7 +9,6 @@ unless ARGV.first
   USAGE
   exit 1
 end
-namespaces = Hash[ARGV[1..-1].map { |ns| ns.split('=') }]
 
 def syntax_results(results)
   results.select do |res|
@@ -25,9 +24,18 @@ end
 
 require 'bel_parser'
 require 'bel_parser/expression/validator'
-require 'bel_parser/resource/resource_file_reader'
+require 'bel_parser/resource/resource_url_reader'
+
+resource_reader = BELParser::Resource::ResourceURLReader.new(true)
+namespaces      =
+  Hash[
+    ARGV.map do |ns|
+      prefix, identifier = ns.split('=')
+      dataset            = resource_reader.retrieve_resource(identifier)
+      dataset ? [prefix, dataset] : nil
+    end.compact
+  ]
 
-resource_reader = BELParser::Resource::ResourceFileReader.new
 BELParser::Expression::Validator
   .new(ARGV.first, namespaces, resource_reader)
   .each($stdin) do |(line_number, line, ast, results)|

data/bin/bel_script_reader

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(
 
 require 'optparse'
 require 'bel_parser'
-require 'bel_parser/resource/resource_file_reader'
+require 'bel_parser/resource/resource_url_reader'
 
 options = {
   spec: BELParser::Language.specification(
@@ -37,6 +37,14 @@ OptionParser.new do |opts|
 
     options[:spec] = BELParser::Language.specification(spec)
   end
+
+  opts.on(
+    '-r',
+    '--[no-]reuse-resource-databases',
+    'Enable to reuse resource databases if they exist.') do |reuse|
+
+    options[:reuse] = reuse
+  end
 end.parse!
 
 file, spec = options.values_at(:file, :spec)
@@ -70,7 +78,7 @@ SYN_ERR  = BELParser::Language::Syntax::SyntaxError
 SYN_WARN = BELParser::Language::Syntax::SyntaxWarning
 SEM_WARN = BELParser::Language::Semantics::SemanticsWarning
 
-rr         = BELParser::Resource::ResourceFileReader.new
+rr = BELParser::Resource::ResourceURLReader.new(options[:reuse])
 namespaces = Hash[
   ARGV.map do |ns|
     prefix, identifier = ns.split('=')
@@ -80,7 +88,7 @@ namespaces = Hash[
 ]
 
 initial_state = {
-  resource_reader:       BELParser::Resource::ResourceFileReader.new,
+  resource_reader:       rr,
   specification:         spec,
   namespace_definitions: namespaces
 }

data/lib/bel_parser/expression/validator.rb

@@ -49,9 +49,9 @@ if __FILE__ == $PROGRAM_NAME
     exit 1
   end
 
-  require 'bel_parser/resource/resource_file_reader'
+  require 'bel_parser/resource/resource_url_reader'
 
-  resource_reader = BELParser::Resource::ResourceFileReader.new
+  resource_reader = BELParser::Resource::ResourceURLReader.new
   namespaces      =
     Hash[
       ARGV[1..-1]

data/lib/bel_parser/resource/eager_reader.rb

@@ -15,61 +15,75 @@ module BELParser
         @locks ||= Concurrent::Hash.new
       end
 
+      def load_threads
+        @load_threads ||= Concurrent::Hash.new
+      end
+
+      def load_values(identifier)
+        lock = locks[identifier] ||= Mutex.new
+        if identifier.include?('taxonomy')
+          puts "load_all_values, try lock..."
+            end
+        if lock.try_lock
+          load_threads[identifier] = Thread.new do
+            value_hash = {
+              :name       => {},
+              :identifier => {},
+              :title      => {},
+              :synonyms   => {}
+            }
+            retrieve_values_from_resource(identifier).each do |concept|
+              value_hash[:name][concept.name]             = concept
+              value_hash[:identifier][concept.identifier] = concept
+              value_hash[:title][concept.title]           = concept
+              concept.synonyms.each do |synonym|
+                value_hash[:synonyms][synonym] = concept
+              end
+            end
+            resources[identifier] = value_hash
+            lock.unlock
+          end
+        end
+      end
+
       def retrieve_resource(resource_identifier)
         if !resources.key?(resource_identifier)
           load_all_values(resource_identifier)
         end
-        super
       end
 
       def retrieve_value_from_resource(resource_identifier, value)
         if !resources.key?(resource_identifier)
           load_all_values(resource_identifier)
+          if load_threads.key?(resource_identifier)
+            load_thread = load_threads[resource_identifier]
+            load_thread.join unless load_thread == Thread.current
+          end
         end
 
         concepts = resources[resource_identifier]
-        return super unless concepts
+        return nil unless concepts
         CACHE_KEYS.each do |key|
           cached_concept = concepts[key].fetch(value, nil)
           return cached_concept if cached_concept
         end
-        
         nil
       end
 
       def retrieve_values_from_resource(resource_identifier)
         if !resources.key?(resource_identifier)
           load_all_values(resource_identifier)
+          if load_threads.key?(resource_identifier)
+            load_thread = load_threads[resource_identifier]
+            load_thread.join unless load_thread == Thread.current
+          end
         end
 
         concepts = resources[resource_identifier]
-        return super unless concepts
+        return nil unless concepts
         concepts[:name].values
       end
 
-      def load_all_values(identifier)
-        lock = locks[identifier] ||= Mutex.new
-        if lock.try_lock
-          Thread.new do
-            value_hash = {
-              :name       => {},
-              :identifier => {},
-              :title      => {},
-              :synonyms   => {}
-            }
-            retrieve_values_from_resource(identifier).each do |concept|
-              value_hash[:name][concept.name]             = concept
-              value_hash[:identifier][concept.identifier] = concept
-              value_hash[:title][concept.title]           = concept
-              concept.synonyms.each do |synonym|
-                value_hash[:synonyms][synonym] = concept
-              end
-            end
-            resources[identifier] = value_hash
-            lock.unlock
-          end
-        end
-      end
     end
   end
 end

data/lib/bel_parser/resource/file_resource.rb

@@ -0,0 +1,21 @@
+require_relative 'dataset'
+
+# FileResource is a {Dataset} description obtained from a Annotation or
+# Namespace file.
+module BELParser
+  module Resource
+    class FileResource
+      include Dataset
+
+      attr_reader :identifier, :domain, :keyword, :name, :types
+
+      def initialize(identifier, domain, keyword, name, type)
+        @identifier = identifier.to_s
+        @domain     = domain.to_s
+        @keyword    = keyword.to_s
+        @name       = name.to_s
+        @types      = [type]
+      end
+    end
+  end
+end

data/lib/bel_parser/resource/file_resource_value.rb

@@ -0,0 +1,24 @@
+require_relative 'value'
+
+# FileResourceValue is a {Value} obtained from a Annotation or Namespace
+# file.
+module BELParser
+  module Resource
+    class FileResourceValue
+      include Value
+
+      attr_reader :dataset, :name, :encodings
+
+      def initialize(dataset, name, encodings)
+        @dataset   = dataset
+        @name      = name
+        @encodings =
+          if dataset.annotation_resource? && !dataset.namespace_resource?
+            nil
+          else
+            encodings.chars.map(&:to_sym)
+          end
+      end
+    end
+  end
+end

data/lib/bel_parser/resource/lru_cache.rb

@@ -3,7 +3,7 @@ require 'concurrent/hash'
 module BELParser
   module Resource
     # LRUCache implements a least recently used cache. This implementation was
-    # initially copied from https://github.com/SamSaffron/lru_redux.
+    # adapted from https://github.com/SamSaffron/lru_redux.
 		class LRUCache
 			def initialize(max_size)
 				raise ArgumentError.new(:max_size) if max_size < 1

data/lib/bel_parser/resource/lru_reader.rb

@@ -4,30 +4,29 @@ module BELParser
   module Resource
     module LRUReader
 
-      LRU_MAX_SIZE = 500
+      LRU_MAX_SIZE = 2000
       private_constant :LRU_MAX_SIZE
 
-      def resources
-        @resources ||= Concurrent::Hash.new
-      end
-
       def retrieve_resource(resource_identifier)
-        if !resources.key?(resource_identifier)
-          resources[resource_identifier] = LRUCache.new(LRU_MAX_SIZE)
-        end
-        super
+        @resources ||= Hash.new { |hash, key| hash[key] = {} }
+        cached_dataset = @resources[resource_identifier][:dataset]
+        return cached_dataset if cached_dataset
+
+        resolved_dataset = super
+        @resources[resource_identifier][:dataset] = resolved_dataset
+        @resources[resource_identifier][:values]  = LRUCache.new(LRU_MAX_SIZE)
+        resolved_dataset
       end
 
       def retrieve_value_from_resource(resource_identifier, value)
-        if !resources.key?(resource_identifier)
-          resources[resource_identifier] = LRUCache.new(LRU_MAX_SIZE)
-        end
-
-        concepts = resources[resource_identifier]
-        concepts.getset(value) { super }
+        retrieve_resource(resource_identifier)
+        @resources[resource_identifier][:values].getset(value) {
+          super
+        }
       end
 
       def retrieve_values_from_resource(resource_identifier)
+        retrieve_resource(resource_identifier)
         super
       end
     end

data/lib/bel_parser/resource/resource_url_reader.rb

@@ -0,0 +1,160 @@
+require          'base64'
+require          'digest'
+require          'gdbm'
+require          'net/http'
+require          'tempfile'
+require          'uri'
+require_relative 'reader'
+require_relative 'file_resource'
+require_relative 'file_resource_value'
+
+module BELParser
+  module Resource
+    # ResourceURLReader retrieves {Dataset datasets} and {Value values} from
+    # Annotation (i.e. belanno extension) and Namespace (i.e. belns extension)
+    # files. Value and encoding are stored in GDBM database files to reduce the
+    # runtime memory usage (22 resources loaded, totaling 100MB memory usage).
+    #
+    # Only supports resource identifiers with an HTTP or HTTPS scheme.
+    class ResourceURLReader
+      include Reader
+
+      DEFAULT_RESOURCE_VALUE_DELIMITER = '|'
+      private_constant :DEFAULT_RESOURCE_VALUE_DELIMITER
+
+      def initialize(reuse_database_files = false)
+        @resources = {}
+        @datasets  = ::GDBM.new(_temporary_datasets_file)
+        @reuse     = reuse_database_files
+      end
+
+      def retrieve_resource(resource_identifier)
+        read_resource(resource_identifier)[:dataset]
+      end
+
+      def retrieve_value_from_resource(resource_identifier, value)
+        resource = read_resource(resource_identifier)
+        encoding = resource[:values][value]
+        return nil unless encoding
+        FileResourceValue.new(resource[:dataset], value, encoding)
+      end
+
+      def retrieve_values_from_resource(resource_identifier)
+        resource = read_resource(resource_identifier)
+        dataset  = resource[:dataset]
+        resource[:values].lazy.map do |value, encoding|
+          FileResourceValue.new(dataset, value, encoding)
+        end
+      end
+
+      protected
+
+      def read_resource(url)
+        # return cached resource if present
+        resource = @resources[url]
+        return resource if resource
+
+        # read url
+        content = http_get(url)
+        unless content
+          # cache and return empty resource if not resolvable
+          return @resources[url] = empty_resource
+        end
+
+        @resources[url] = create_resource(url, content.each_line)
+      end
+
+      def create_resource(url, line_enum)
+        delimiter           = DEFAULT_RESOURCE_VALUE_DELIMITER
+        dataset             = @datasets[url]
+        value_database_file = _temporary_database_file(url)
+        values              = ::GDBM.new(value_database_file)
+
+        if @reuse && dataset && values.size > 0
+          warn(
+            <<-MSG.gsub(/^ {14}/, '')
+              Warning - Reusing value database.
+                URL:  #{url}
+                File: #{value_database_file}
+            MSG
+          )
+          return {
+            dataset: FileResource.new(url, *dataset.split('//')),
+            values:  values
+          }
+        end
+
+        value_section               = false
+        type, name, keyword, domain = nil
+        line_enum.each do |line|
+          line.strip!
+          case
+          when line =~ /^#{Regexp.escape('[AnnotationDefinition]')}/
+            type = Dataset::ANNOTATION
+          when line =~ /^#{Regexp.escape('[Namespace]')}/
+            type = Dataset::NAMESPACE
+          when line =~ /^NameString *= *(.*)$/
+            name = Regexp.last_match[1]
+          when line =~ /^Keyword *= *(.*)$/
+            keyword = Regexp.last_match[1]
+          when line =~ /^DomainString *= *(.*)$/
+            domain = Regexp.last_match[1]
+          when line =~ /^DelimiterString *=(.*)$/
+            delimiter = Regexp.last_match[1]
+          when line =~ /^#{Regexp.escape('[Values]')}/
+            dataset = FileResource.new(url, domain, keyword, name, type)
+            value_section = true
+          when value_section
+            value, encoding    = line.strip.split(delimiter)
+            values[value.to_s] = encoding.to_s
+          end
+        end
+
+        resource = {
+          dataset: FileResource.new(url, domain, keyword, name, type),
+          values:  values
+        }
+        @datasets[url] = [domain, keyword, name, type].join('//')
+        resource
+      end
+
+      def empty_resource
+        { dataset: nil, values: {} }
+      end
+
+      def http_get(url)
+        begin
+          _get(URI.parse(url))
+        rescue URI::InvalidURIError, SocketError
+          return nil
+        end
+      end
+
+      private
+
+      def _get(url, &block)
+        Net::HTTP.start(url.host, url.port) do |http|
+          http.request(Net::HTTP::Get.new(url)) do |response|
+            return response.read_body
+          end
+        end
+      end
+
+      def _hash_url(url)
+        Base64.encode64(Digest::SHA1.digest(url)).delete("/=\n")
+      end
+
+      def _temporary_datasets_file
+        resource_directory = File.join(Dir.tmpdir, 'belresources')
+        FileUtils.mkdir_p(resource_directory)
+        File.join(resource_directory, 'datasets.gdbm')
+      end
+
+      def _temporary_database_file(url)
+        resource_directory = File.join(Dir.tmpdir, 'belresources')
+        FileUtils.mkdir_p(resource_directory)
+        File.join(resource_directory, "#{_hash_url(url)}.gdbm")
+      end
+    end
+  end
+end

data/lib/bel_parser/script/validator.rb

@@ -98,10 +98,12 @@ end
 if __FILE__ == $PROGRAM_NAME
   $LOAD_PATH.unshift(
     File.join(File.expand_path(File.dirname(__FILE__)), '..', '..'))
+
   require 'bel_parser/language'
-  require 'bel_parser/resource/resource_file_reader'
+  require 'bel_parser/resource/resource_url_reader'
+
   initial_state = {
-    resource_reader: BELParser::Resource::ResourceFileReader.new,
+    resource_reader: BELParser::Resource::ResourceURLReader.new,
     specification:   BELParser::Language.specification(
       BELParser::Language.latest_supported_version
     )

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bel_parser
 version: !ruby/object:Gem::Version
-  version: 1.0.0.alpha.16
+  version: 1.0.0.alpha.17
 platform: ruby
 authors:
 - Anthony Bargnesi
@@ -304,12 +304,13 @@ files:
 - lib/bel_parser/resource/dataset.rb
 - lib/bel_parser/resource/eager_reader.rb
 - lib/bel_parser/resource/eager_sparql_reader.rb
-- lib/bel_parser/resource/http_cache.rb
+- lib/bel_parser/resource/file_resource.rb
+- lib/bel_parser/resource/file_resource_value.rb
 - lib/bel_parser/resource/jena_tdb_reader.rb
 - lib/bel_parser/resource/lru_cache.rb
 - lib/bel_parser/resource/lru_reader.rb
 - lib/bel_parser/resource/reader.rb
-- lib/bel_parser/resource/resource_file_reader.rb
+- lib/bel_parser/resource/resource_url_reader.rb
 - lib/bel_parser/resource/sparql_reader.rb
 - lib/bel_parser/resource/value.rb
 - lib/bel_parser/script.rb

data/lib/bel_parser/resource/http_cache.rb

@@ -1,89 +0,0 @@
-require 'base64'
-require 'concurrent/hash'
-require 'digest'
-require 'fileutils'
-require 'net/http'
-require 'tempfile'
-
-module BELParser
-  module Resource
-    class HTTPCache
-
-      # Lock (per URL) ensures concurrent retrieval/caching is thread-safe.
-      URL_LOCKS = Concurrent::Hash.new do |hash, key|
-        hash[key] = Mutex.new
-      end
-      private_constant :URL_LOCKS
-
-      def initialize(base_dir="#{Dir.tmpdir}/bel-resources")
-        @base_dir  = FileUtils.mkdir_p(base_dir).first
-      end
-
-      def get(url)
-        cached_url_path = File.join(@base_dir, hash_url(url))
-        URL_LOCKS[url].synchronize do
-          if File.exists?(cached_url_path)
-            if block_given?
-              File.open(cached_url_path) do |cached_file|
-                yield cached_file
-              end
-            else
-              File.open(cached_url_path)
-            end
-          else
-            uri = URI.parse(url)
-            begin
-              Net::HTTP.start(uri.host, uri.port) do |http|
-                http.request(Net::HTTP::Get.new(uri)) do |response|
-
-                  if block_given?
-                    if response.is_a?(Net::HTTPOK)
-                      cached_file = File.open(cached_url_path, 'w')
-                      response.read_body do |chunk|
-                        cached_file.write(chunk)
-                        yield StringIO.new(chunk)
-                      end
-                    else
-                      yield nil
-                    end
-                  else
-                    if response.is_a?(Net::HTTPOK)
-                      cached_file = File.open(cached_url_path, 'w')
-                      content = response.read_body
-                      cached_file.write(content)
-                      return StringIO.new(content)
-                    else
-                      return nil
-                    end
-                  end
-                end
-              end
-            rescue SocketError
-              if block_given?
-                yield nil
-              else
-                return nil
-              end
-            ensure
-              if defined? cached_file
-                cached_file.close
-              end
-            end
-          end
-        end
-      end
-
-      protected
-
-      def hash_url(url)
-        Base64.encode64(Digest::SHA1.digest(url)).delete("/=\n")
-      end
-    end
-  end
-end
-
-if __FILE__ == $PROGRAM_NAME
-  BELParser::Resource::HTTPCache.new.get(ARGV.first) do |io|
-    io.each_line { |line| puts line }
-  end
-end

data/lib/bel_parser/resource/resource_file_reader.rb

@@ -1,135 +0,0 @@
-require 'open-uri'
-require_relative 'reader'
-require_relative 'eager_reader'
-require_relative 'dataset'
-require_relative 'http_cache'
-require_relative 'value'
-
-module BELParser
-  module Resource
-    # ResourceFileReader retrieves {Dataset datasets} and {Value values} from
-    # Annotation (i.e. belanno extension) and Namespace (i.e. belns extension)
-    # files.
-    #
-    # Only supports resource identifiers with an HTTP or HTTPS scheme.
-    class ResourceFileReader
-      include Reader
-      prepend EagerReader
-
-      DEFAULT_RESOURCE_VALUE_DELIMITER = '|'
-      private_constant :DEFAULT_RESOURCE_VALUE_DELIMITER
-
-      def initialize
-        @http_cache = HTTPCache.new
-      end
-
-      def retrieve_resource(resource_identifier)
-        read_resource(resource_identifier)
-      end
-
-      def retrieve_value_from_resource(resource_identifier, value)
-        file_resource = read_resource(resource_identifier)
-
-        io = @http_cache.get(resource_identifier)
-        delimiter = DEFAULT_RESOURCE_VALUE_DELIMITER
-        io.each_line do |line|
-          case line
-          when /^DelimiterString *=(.*)/
-            delimiter = Regexp.last_match[1]
-          when /^#{Regexp.escape('[Values]')}/
-            break
-          end
-        end
-
-        value_match = "#{value}#{delimiter}"
-        value_line  = io.each_line.find do |line|
-          line.start_with?(value_match)
-        end
-        return nil unless value_line
-        value, encoding = value_line.strip.split(delimiter)
-        FileResourceValue.new(file_resource, value, encoding)
-      end
-
-      def retrieve_values_from_resource(resource_identifier)
-        file_resource = read_resource(resource_identifier)
-
-        io = @http_cache.get(resource_identifier)
-        delimiter = DEFAULT_RESOURCE_VALUE_DELIMITER
-        io.each_line do |line|
-          case line
-          when /^DelimiterString *=(.*)/
-            delimiter = Regexp.last_match[1]
-          when /^#{Regexp.escape('[Values]')}/
-            break
-          end
-        end
-
-        io.each_line.map do |line|
-          value, encoding = line.strip.split(delimiter)
-          FileResourceValue.new(file_resource, value, encoding)
-        end
-      end
-
-      private
-
-      def read_resource(url)
-        io = @http_cache.get(url)
-        return nil unless io
-
-        type, name, keyword, domain = nil
-        io.each_line do |line|
-          case line
-          when /^#{Regexp.escape('[AnnotationDefinition]')}/
-            type = Dataset::ANNOTATION
-          when /^#{Regexp.escape('[Namespace]')}/
-            type = Dataset::NAMESPACE
-          when /^NameString *= *(.*)$/
-            name = Regexp.last_match[1]
-          when /^Keyword *= *(.*)$/
-            keyword = Regexp.last_match[1]
-          when /^DomainString *= *(.*)$/
-            domain = Regexp.last_match[1]
-          when /^#{Regexp.escape('[Values]')}/
-            break
-          end
-        end
-        FileResource.new(url, domain, keyword, name, type)
-      end
-
-      # FileResource is a {Dataset} description obtained from a Annotation or
-      # Namespace file.
-      class FileResource
-        include Dataset
-
-        attr_reader :identifier, :domain, :keyword, :name, :types
-
-        def initialize(identifier, domain, keyword, name, type)
-          @identifier = identifier.to_s
-          @domain     = domain.to_s
-          @keyword    = keyword.to_s
-          @name       = name.to_s
-          @types      = [type]
-        end
-      end
-
-      # FileResourceValue is a {Value} obtained from a Annotation or Namespace
-      # file.
-      class FileResourceValue
-        include Value
-
-        attr_reader :dataset, :name, :encodings
-
-        def initialize(dataset, name, encodings)
-          @dataset   = dataset
-          @name      = name
-          @encodings =
-            if dataset.annotation_resource? && !dataset.namespace_resource?
-              nil
-            else
-              encodings.chars.map(&:to_sym)
-            end
-        end
-      end
-    end
-  end
-end