openbel-api 0.5.1-java → 0.6.1-java

data/app/openbel/api/routes/authenticate.rb

@@ -1,100 +1,8 @@
-require 'uri'
-require 'rest-client'
-
-def current_host(env)
-  scheme = env['rack.url_scheme'] || 'http'
-  host = env['HTTP_HOST']
-  "#{scheme}://#{host}"
-end
-
-def current_path(env)
-  scheme = env['rack.url_scheme'] || 'http'
-  host = env['HTTP_HOST']
-  path = env['PATH_INFO']
-  "#{scheme}://#{host}#{path}"
-end
-
 module OpenBEL
   module Routes
 
     class Authenticate < Base
 
-      get '/api/authenticate' do
-        state = params[:state]
-        code = params[:code]
-        if code.nil?
-          default_connection = OpenBEL::Settings[:auth][:default_connection]
-          default_auth_url = current_path(env) + "/#{default_connection}"
-          if not state.nil?
-            default_auth_url += "?state=#{state}"
-          end
-          redirect to(default_auth_url)
-        end
-
-        domain = OpenBEL::Settings[:auth][:domain]
-        id = OpenBEL::Settings[:auth][:id]
-        secret = OpenBEL::Settings[:auth][:secret]
-
-        callback_url = current_path(env)
-        payload =  {
-            client_id: id,
-            client_secret: secret,
-            redirect_uri: callback_url,
-            code: code,
-            grant_type: :authorization_code
-        }
-
-        token_url = "https://#{domain}/oauth/token"
-        body = payload.to_json
-
-        begin
-          token_response = RestClient.post token_url, body,
-                                           :content_type => :json,
-                                           :accept => :json
-        rescue => e
-          hdrs = {'Content-Type' => 'application/json'}
-          msg = {error: e.response }
-          return [401, hdrs, [msg.to_json]]
-        end
-
-        token_response = JSON.parse(token_response)
-        access_token = token_response['access_token']
-        jwt = token_response['id_token']
-
-        user_url = "https://#{domain}/userinfo?access_token=#{access_token}"
-        begin
-          user_response = RestClient.get user_url, :accept => :json
-        rescue => e
-          hdrs = {'Content-Type' => 'application/json'}
-          msg = {error: e.response }
-          return [401, hdrs, [msg.to_json]]
-        end
-
-        email = JSON.parse(user_response)['email']
-        hdrs = {'Content-Type' => 'application/json'}
-        msg = {success: email}
-        cookies[:jwt] = jwt
-        if not state.nil?
-          redirect to(state + "?jwt=#{jwt}")
-        else
-          [200, hdrs, [msg.to_json]]
-        end
-      end
-
-      get '/api/authenticate/:connection' do
-        state = params[:state]
-        redirect_setting = OpenBEL::Settings[:auth][:redirect]
-        connection = params[:connection]
-        redirect_uri = current_host(env) + '/api/authenticate'
-        auth_url = "#{redirect_setting}"
-        auth_url += "&redirect_uri=#{redirect_uri}"
-        auth_url += "&connection=#{connection}"
-        if not state.nil?
-          auth_url += "&state=#{state}"
-        end
-        redirect to(auth_url)
-      end
-
       get '/api/authentication-enabled' do
         enabled = OpenBEL::Settings[:auth][:enabled]
         hdrs = {'Content-Type' => 'application/json'}

data/app/openbel/api/routes/base.rb

@@ -21,14 +21,14 @@ module OpenBEL
       include OpenBEL::Resource::Namespaces
       include OpenBEL::Schemas
 
-      DEFAULT_CONTENT_TYPE   = 'application/hal+json'
-      SPOKEN_CONTENT_TYPES   = %w[application/hal+json application/json]
+      DEFAULT_CONTENT_TYPE    = 'application/hal+json'
+      DEFAULT_CONTENT_TYPE_ID = :hal
+      SPOKEN_CONTENT_TYPES    = %w[application/hal+json application/json]
       SPOKEN_CONTENT_TYPES.concat(
         BEL::Translator.plugins.values.flat_map { |p| p.media_types.map(&:to_s) }
       )
-
-      SCHEMA_BASE_URL        = 'http://next.belframework.org/schemas/'
-      RESOURCE_SERIALIZERS   = {
+      SCHEMA_BASE_URL         = 'http://next.belframework.org/schemas/'
+      RESOURCE_SERIALIZERS    = {
         :annotation                 => AnnotationResourceSerializer,
         :annotation_collection      => AnnotationCollectionSerializer,
         :annotation_value           => AnnotationValueResourceSerializer,
@@ -80,6 +80,16 @@ module OpenBEL
           SCHEMA_BASE_URL + "#{name}.schema.json"
         end
 
+        def wants_default?
+          if params[:format]
+            return params[:format] == DEFAULT_CONTENT_TYPE
+          end
+
+          request.accept.any? { |accept_entry|
+            accept_entry.to_s == DEFAULT_CONTENT_TYPE
+          }
+        end
+
         def validate_media_type!(content_type, options = {})
           ctype = request.content_type
           valid = ctype.start_with? content_type
@@ -143,6 +153,10 @@ module OpenBEL
           end
         end
 
+        def write_filter(filter)
+          MultiJson.dump(filter)
+        end
+
         def render_json(obj, media_type = 'application/hal+json', profile = nil)
           ctype =
             if profile

data/app/openbel/api/routes/datasets.rb

@@ -1,10 +1,13 @@
 require 'bel'
+require 'bel/util'
 require 'rdf'
 require 'cgi'
 require 'multi_json'
 require 'openbel/api/evidence/mongo'
 require 'openbel/api/evidence/facet_filter'
 require_relative '../resources/evidence_transform'
+require_relative '../helpers/evidence'
+require_relative '../helpers/filters'
 require_relative '../helpers/pager'
 
 module OpenBEL
@@ -16,7 +19,6 @@ module OpenBEL
       include OpenBEL::Helpers
 
       DEFAULT_TYPE = 'application/hal+json'
-
       ACCEPTED_TYPES = {
         :bel  => 'application/bel',
         :xml  => 'application/xml',
@@ -24,9 +26,14 @@ module OpenBEL
         :json => 'application/json',
       }
 
+      MONGO_BATCH     = 500
+      FACET_THRESHOLD = 10000
+
       def initialize(app)
         super
 
+        BEL.translator(:rdf)
+
         # Evidence API using Mongo.
         mongo = OpenBEL::Settings[:evidence_store][:mongo]
         @api  = OpenBEL::Evidence::Evidence.new(mongo)
@@ -36,9 +43,6 @@ module OpenBEL
           :tdb_directory => OpenBEL::Settings[:resource_rdf][:jena][:tdb_directory]
         )
 
-        # Load RDF monkeypatches.
-        BEL::Translator.plugins[:rdf].create_translator
-
         # Annotations using RdfRepository
         annotations = BEL::Resource::Annotations.new(@rr)
         @annotation_transform = AnnotationTransform.new(annotations)
@@ -233,35 +237,66 @@ the "multipart/form-data" content type. Allowed dataset content types are: #{ACC
         # Create dataset in RDF.
         @rr.insert_statements(void_dataset)
 
-        dataset = retrieve_dataset(void_dataset_uri)
+        dataset    = retrieve_dataset(void_dataset_uri)
+        dataset_id = dataset[:identifier]
+
+        # Add batches of read evidence objects; save to Mongo and RDF.
+        # TODO Add JRuby note regarding Enumerator threading.
+        evidence_count = 0
+        evidence_batch = []
 
-        # Add slices of read evidence objects; save to Mongo and RDF.
-        BEL.evidence(io, type).each.lazy.each_slice(500) do |slice|
-          slice.map! do |ev|
-            # Standardize annotations from experiment_context.
-            @annotation_transform.transform_evidence!(ev, base_url)
+        # Clear out all facets before loading dataset.
+        @api.delete_facets
 
-            # Add filterable metadata field for dataset identifier.
-            ev.metadata[:dataset] = dataset[:identifier]
+        BEL.evidence(io, type).each do |ev|
+          # Standardize annotations from experiment_context.
+          @annotation_transform.transform_evidence!(ev, base_url)
 
-            facets           = map_evidence_facets(ev)
-            ev.bel_statement = ev.bel_statement.to_s
-            hash             = ev.to_h
-            hash[:facets]    = facets
+          ev.metadata[:dataset] = dataset_id
+          facets                = map_evidence_facets(ev)
+          ev.bel_statement      = ev.bel_statement.to_s
+          hash                  = BEL.object_convert(String, ev.to_h) { |str|
+                                    str.gsub(/\n/, "\\n").gsub(/\r/, "\\r")
+                                  }
+          hash[:facets]         = facets
+          # Create dataset field for efficient removal.
+          hash[:_dataset]       = dataset_id
 
-            # Create dataset field for efficient removal.
-            hash[:_dataset]  = dataset[:identifier]
-            hash
+          evidence_batch << hash
+
+          if evidence_batch.size == MONGO_BATCH
+            _ids = @api.create_evidence(evidence_batch)
+
+            dataset_parts = _ids.map { |object_id|
+              RDF::Statement.new(void_dataset_uri, RDF::DC.hasPart, object_id.to_s)
+            }
+            @rr.insert_statements(dataset_parts)
+
+            evidence_batch.clear
+
+            # Clear out all facets after FACET_THRESHOLD nanopubs have been seen.
+            evidence_count += MONGO_BATCH
+            if evidence_count >= FACET_THRESHOLD
+              @api.delete_facets
+              evidence_count = 0
+            end
           end
+        end
 
-          _ids = @api.create_evidence(slice)
+        unless evidence_batch.empty?
+          _ids = @api.create_evidence(evidence_batch)
 
           dataset_parts = _ids.map { |object_id|
             RDF::Statement.new(void_dataset_uri, RDF::DC.hasPart, object_id.to_s)
           }
           @rr.insert_statements(dataset_parts)
+
+          evidence_batch.clear
         end
 
+        # Clear out all facets after the dataset is completely loaded.
+        @api.delete_facets
+
         status 201
         headers 'Location' => void_dataset_uri.to_s
       end
@@ -297,142 +332,19 @@ the "multipart/form-data" content type. Allowed dataset content types are: #{ACC
         start                = (params[:start]  || 0).to_i
         size                 = (params[:size]   || 0).to_i
         faceted              = as_bool(params[:faceted])
-        max_values_per_facet = (params[:max_values_per_facet] || 0).to_i
-
-        # check filters
-        filters = []
-        filter_params = CGI::parse(env["QUERY_STRING"])['filter']
-        filter_params.each do |filter|
-          filter = read_filter(filter)
-          halt 400 unless ['category', 'name', 'value'].all? { |f| filter.include? f}
-
-          if filter['category'] == 'fts' && filter['name'] == 'search'
-            unless filter['value'].to_s.length > 1
-              halt(
-                400,
-                { 'Content-Type' => 'application/json' },
-                render_json({
-                  :status => 400,
-                  :msg => 'Full-text search filter values must be larger than one.'
-                })
-              )
-            end
-          end
-
-          # Remove dataset filters since we're filtering a specific one already.
-          next if filter.values_at('category', 'name') == ['metadata', 'dataset']
+        max_values_per_facet = (params[:max_values_per_facet] || -1).to_i
 
-          filters << filter
-        end
+        filters = validate_filters!
 
         collection_total  = @api.count_evidence
         filtered_total    = @api.count_evidence(filters)
-        page_results      = @api.find_dataset_evidence(dataset, filters, start, size, faceted)
-
-        accept_type = request.accept.find { |accept_entry|
-          ACCEPTED_TYPES.values.include?(accept_entry.to_s)
-        }
-        accept_type ||= DEFAULT_TYPE
-
-        if params[:format]
-          translator  = BEL::Translator.plugins[params[:format].to_sym]
-          halt 501 if !translator || translator.id == :rdf
-          accept_type = [translator.media_types].flatten.first
-        end
-
-        if accept_type == DEFAULT_TYPE
-          evidence          = page_results[:cursor].map { |item|
-            item.delete('facets')
-            item
-          }.to_a
-
-          facets            = page_results[:facets]
-
-          halt 404 if evidence.empty?
-
-          pager = Pager.new(start, size, filtered_total)
-
-          options = {
-            :start    => start,
-            :size     => size,
-            :filters  => filter_params,
-            :metadata => {
-              :collection_paging => {
-                :total                  => collection_total,
-                :total_filtered         => pager.total_size,
-                :total_pages            => pager.total_pages,
-                :current_page           => pager.current_page,
-                :current_page_size      => evidence.size,
-              }
-            }
-          }
-
-          if facets
-            # group by category/name
-            hashed_values = Hash.new { |hash, key| hash[key] = [] }
-            facets.each { |facet|
-              filter         = read_filter(facet['_id'])
-              category, name = filter.values_at('category', 'name')
-              next if !category || !name
-
-              key = [category.to_sym, name.to_sym]
-              facet_obj = {
-                :value    => filter['value'],
-                :filter   => facet['_id'],
-                :count    => facet['count']
-              }
-              hashed_values[key] << facet_obj
-            }
-
-            if max_values_per_facet == 0
-              facet_hashes = hashed_values.map { |(category, name), value_objects|
-                {
-                  :category => category,
-                  :name     => name,
-                  :values   => value_objects
-                }
-              }
-            else
-              facet_hashes = hashed_values.map { |(category, name), value_objects|
-                {
-                  :category => category,
-                  :name     => name,
-                  :values   => value_objects.take(max_values_per_facet)
-                }
-              }
-            end
-
-            options[:facets] = facet_hashes
-          end
-
-          # pager links
-          options[:previous_page] = pager.previous_page
-          options[:next_page]     = pager.next_page
-
-          render_collection(evidence, :evidence, options)
-        else
-          out_translator = BEL.translator(accept_type)
-          extension      = ACCEPTED_TYPES.key(accept_type.to_s)
-
-          response.headers['Content-Type'] = accept_type
-          status 200
-          attachment "#{dataset[:identifier].gsub(/[^\w]/, '_')}.#{extension}"
-          stream :keep_open do |response|
-            cursor             = page_results[:cursor]
-            json_evidence_enum = cursor.lazy.map { |evidence|
-              evidence.delete('facets')
-              evidence.delete('_id')
-              evidence.keys.each do |key|
-                evidence[key.to_sym] = evidence.delete(key)
-              end
-              BEL::Model::Evidence.create(evidence)
-            }
+        page_results      = @api.find_dataset_evidence(dataset, filters, start, size, faceted, max_values_per_facet)
+        name              = dataset[:identifier].gsub(/[^\w]/, '_')
 
-            out_translator.write(json_evidence_enum) do |converted_evidence|
-              response << converted_evidence
-            end
-          end
-        end
+        render_evidence_collection(
+          name, page_results, start, size, filters,
+          filtered_total, collection_total, @api
+        )
       end
 
       get '/api/datasets' do
@@ -469,6 +381,7 @@ the "multipart/form-data" content type. Allowed dataset content types are: #{ACC
         halt 404 unless dataset_exists?(void_dataset_uri)
 
         dataset = retrieve_dataset(void_dataset_uri)
+        # XXX Removes all facets due to load of many evidence.
         @api.delete_dataset(dataset[:identifier])
         @rr.delete_statement(RDF::Statement.new(void_dataset_uri, nil, nil))
 
@@ -485,6 +398,7 @@ the "multipart/form-data" content type. Allowed dataset content types are: #{ACC
 
         datasets.each do |void_dataset_uri|
           dataset = retrieve_dataset(void_dataset_uri)
+          # XXX Removes all facets due to load of many evidence.
           @api.delete_dataset(dataset[:identifier])
           @rr.delete_statement(RDF::Statement.new(void_dataset_uri, nil, nil))
         end

data/app/openbel/api/routes/evidence.rb

@@ -3,6 +3,8 @@ require 'cgi'
 require 'openbel/api/evidence/mongo'
 require 'openbel/api/evidence/facet_filter'
 require_relative '../resources/evidence_transform'
+require_relative '../helpers/evidence'
+require_relative '../helpers/filters'
 require_relative '../helpers/pager'
 
 module OpenBEL
@@ -142,19 +144,7 @@ module OpenBEL
         size                 = (params[:size]  || 0).to_i
         group_as_array       = as_bool(params[:group_as_array])
 
-        # check filters
-        filters = []
-        filter_params = CGI::parse(env["QUERY_STRING"])['filter']
-        filter_params.each do |filter|
-          filter = read_filter(filter)
-          halt 400 unless ['category', 'name', 'value'].all? { |f| filter.include? f}
-
-          if filter['category'] == 'fts' && filter['name'] == 'search'
-            halt 400 unless filter['value'].to_s.length > 1
-          end
-
-          filters << filter
-        end
+        filters = validate_filters!
 
         cursor  = @api.find_evidence(filters, start, size, false)[:cursor]
         if group_as_array
@@ -168,93 +158,18 @@ module OpenBEL
         start                = (params[:start]  || 0).to_i
         size                 = (params[:size]   || 0).to_i
         faceted              = as_bool(params[:faceted])
-        max_values_per_facet = (params[:max_values_per_facet] || 0).to_i
+        max_values_per_facet = (params[:max_values_per_facet] || -1).to_i
 
-        # check filters
-        filters = []
-        filter_params = CGI::parse(env["QUERY_STRING"])['filter']
-        filter_params.each do |filter|
-          filter = read_filter(filter)
-          halt 400 unless ['category', 'name', 'value'].all? { |f| filter.include? f}
-
-          if filter['category'] == 'fts' && filter['name'] == 'search'
-            halt 400 unless filter['value'].to_s.length > 1
-          end
-
-          filters << filter
-        end
+        filters = validate_filters!
 
         collection_total  = @api.count_evidence()
         filtered_total    = @api.count_evidence(filters)
-        page_results      = @api.find_evidence(filters, start, size, faceted)
-        evidence          = page_results[:cursor].map { |item|
-          item.delete('facets')
-          item
-        }.to_a
-        facets            = page_results[:facets]
-
-        halt 404 if evidence.empty?
-
-        pager = Pager.new(start, size, filtered_total)
-
-        options = {
-          :start    => start,
-          :size     => size,
-          :filters  => filter_params,
-          :metadata => {
-            :collection_paging => {
-              :total                  => collection_total,
-              :total_filtered         => pager.total_size,
-              :total_pages            => pager.total_pages,
-              :current_page           => pager.current_page,
-              :current_page_size      => evidence.size,
-            }
-          }
-        }
-
-        if facets
-          # group by category/name
-          hashed_values = Hash.new { |hash, key| hash[key] = [] }
-          facets.each { |facet|
-            filter         = read_filter(facet['_id'])
-            category, name = filter.values_at('category', 'name')
-            next if !category || !name
-
-            key = [category.to_sym, name.to_sym]
-            facet_obj = {
-              :value    => filter['value'],
-              :filter   => facet['_id'],
-              :count    => facet['count']
-            }
-            hashed_values[key] << facet_obj
-          }
-
-          if max_values_per_facet == 0
-            facet_hashes = hashed_values.map { |(category, name), value_objects|
-              {
-                :category => category,
-                :name     => name,
-                :values   => value_objects
-              }
-            }
-          else
-            facet_hashes = hashed_values.map { |(category, name), value_objects|
-              {
-                :category => category,
-                :name     => name,
-                :values   => value_objects.take(max_values_per_facet)
-              }
-            }
-          end
-
-          options[:facets] = facet_hashes
-        end
-
-        # pager links
-        options[:previous_page] = pager.previous_page
-        options[:next_page]     = pager.next_page
+        page_results      = @api.find_evidence(filters, start, size, faceted, max_values_per_facet)
 
-        render_collection(evidence, :evidence, options)
+        render_evidence_collection(
+          'evidence-export', page_results, start, size, filters,
+          filtered_total, collection_total, @api
+        )
       end
 
       get '/api/evidence/:id' do