openbel-api 0.5.1-java → 0.6.1-java

Sign up to get free protection for your applications and to get access to all the features.
Files changed (23) hide show
  1. checksums.yaml +4 -4
  2. data/.gemspec +14 -6
  3. data/CHANGELOG.md +32 -0
  4. data/README.md +2 -2
  5. data/VERSION +1 -1
  6. data/app/openbel/api/config.rb +12 -0
  7. data/app/openbel/api/helpers/base.rb +17 -0
  8. data/app/openbel/api/helpers/evidence.rb +74 -0
  9. data/app/openbel/api/helpers/filters.rb +94 -0
  10. data/app/openbel/api/helpers/translators.rb +73 -0
  11. data/app/openbel/api/resources/evidence_transform.rb +1 -13
  12. data/app/openbel/api/routes/authenticate.rb +0 -92
  13. data/app/openbel/api/routes/base.rb +19 -5
  14. data/app/openbel/api/routes/datasets.rb +65 -151
  15. data/app/openbel/api/routes/evidence.rb +10 -95
  16. data/config/config.yml +4 -6
  17. data/lib/openbel/api/evidence/api.rb +8 -0
  18. data/lib/openbel/api/evidence/facet_filter.rb +3 -3
  19. data/lib/openbel/api/evidence/mongo.rb +304 -44
  20. data/lib/openbel/api/evidence/mongo_facet.rb +173 -66
  21. metadata +112 -26
  22. data/config/async_evidence.rb +0 -12
  23. data/config/async_jena.rb +0 -14
data/lib/openbel/api/evidence/mongo_facet.rb CHANGED
@@ -12,10 +12,15 @@ module OpenBEL
12
12
  include FacetFilter
13
13
 
14
14
  def initialize(options = {})
15
- host = options[:host]
16
- port = options[:port]
17
- db = options[:database]
18
- @db = MongoClient.new(host, port).db(db)
15
+ host = options[:host]
16
+ port = options[:port]
17
+ db = options[:database]
18
+ @db = MongoClient.new(
19
+ host,
20
+ port,
21
+ :op_timeout => nil,
22
+ :pool_size => 30
23
+ ).db(db)
19
24
 
20
25
  # Authenticate user if provided.
21
26
  username = options[:username]
@@ -25,90 +30,192 @@ module OpenBEL
25
30
  @db.authenticate(username, password, nil, auth_db)
26
31
  end
27
32
 
28
- @evidence = @db[:evidence]
29
- @evidence_facets = @db[:evidence_facets]
33
+ @evidence = @db[:evidence]
34
+ @evidence_facet_cache = @db[:evidence_facet_cache]
35
+
36
+ # ensure all indexes are created and maintained
37
+ ensure_all_indexes
30
38
  end
31
39
 
32
- def create_facets(_id, query_hash)
33
- # create and save facets, identified by query
34
- facets_doc = _id.merge({
35
- :facets => evidence_facets(query_hash)
36
- })
37
- @evidence_facets.save(facets_doc, :j => true)
40
+ def find_facets(query, filters, facet_value_limit = -1)
41
+ sorted_filters = sort_filters(filters)
42
+ cache_collection = facet_cache_collection(sorted_filters)
43
+
44
+ if no_collection?(cache_collection)
45
+ cache_collection = "evidence_facet_cache_#{EvidenceFacets.generate_uuid}"
46
+ create_aggr = create_aggregation(cache_collection, query)
47
+ @evidence.aggregate(create_aggr[:pipeline], create_aggr[:options])
48
+ @evidence_facet_cache.insert({
49
+ :filters => sorted_filters,
50
+ :cache_collection => cache_collection
51
+ })
52
+ end
53
+
54
+ # set up field projection based on value limit
55
+ field_projection = {
56
+ :_id => 0,
57
+ :category => 1,
58
+ :name => 1,
59
+ :values => 1
60
+ }
61
+ if facet_value_limit > 0
62
+ field_projection[:values] = {:$slice => facet_value_limit}
63
+ end
38
64
 
39
- # return facets document
40
- facets_doc
65
+ # cursor facets and apply "filter"
66
+ @db[cache_collection].find({}, :fields => field_projection).map { |facet_doc|
67
+ category, name = facet_doc.values_at('category', 'name')
68
+ facet_doc['values'].each do |value|
69
+ value[:filter] = MultiJson.dump({
70
+ :category => category,
71
+ :name => name,
72
+ :value => value['value']
73
+ })
74
+ end
75
+ facet_doc
76
+ }
41
77
  end
42
78
 
43
- def find_facets(query_hash, filters)
44
- _id = {:_id => to_facets_id(filters)}
45
- @evidence_facets.find_one(_id) || create_facets(_id, query_hash)
79
+ def delete_facets(facets)
80
+ # Add zero-filter to facets; clears the default search
81
+ facets = facets.to_a
82
+ facets.unshift([])
83
+
84
+ # Drop facet cache collections
85
+ @evidence_facet_cache.find(
86
+ {:filters => {:$in => facets}},
87
+ :fields => {:_id => 0, :cache_collection => 1}
88
+ ).each do |doc|
89
+ cache_collection = doc['cache_collection']
90
+ @db[cache_collection].drop()
91
+ end
92
+
93
+ # remove filter match entries in evidence_facet_cache
94
+ @evidence_facet_cache.remove({:filters => {:$in => facets}})
46
95
  end
47
96
 
48
- def remove_facets_by_filters(filters = [], options = {})
49
- remove_spec =
50
- if filters.empty?
51
- { :_id => "" }
52
- else
53
- {
54
- :_id => {
55
- :$in => filters.map { |filter|
56
- to_regexp(MultiJson.load(filter))
57
- }
58
- }
59
- }
60
- end
61
- @evidence_facets.remove(remove_spec, :j => true)
97
+ def delete_all_facets
98
+ @evidence_facet_cache.find(
99
+ {},
100
+ :fields => {:_id => 0, :cache_collection => 1}
101
+ ).each do |doc|
102
+ cache_collection = doc['cache_collection']
103
+ @db[cache_collection].drop()
104
+ end
105
+
106
+ # remove all entries in evidence_facet_cache
107
+ @evidence_facet_cache.remove({})
108
+ end
109
+
110
+ def ensure_all_indexes
111
+ @evidence_facet_cache.ensure_index([
112
+ [:"filters.category", Mongo::ASCENDING],
113
+ [:"filters.name", Mongo::ASCENDING]
114
+ ],
115
+ :background => true
116
+ )
62
117
  end
63
118
 
64
119
  private
65
120
 
66
- def to_regexp(filter)
67
- filter_s = "#{filter['category']}|#{filter['name']}|#{filter['value']}"
68
- %r{.*#{Regexp.escape(filter_s)}.*}
121
+ def no_collection?(collection)
122
+ !collection || !@db.collection_names.include?(collection)
69
123
  end
70
124
 
125
+ def sort_filters(filters)
126
+ filters.sort { |f1, f2|
127
+ f1_array = f1.values_at(:category, :name, :value)
128
+ f2_array = f2.values_at(:cat, :name, :value)
71
129
 
72
- def to_facets_id(filters)
73
- filters.map { |filter|
74
- "#{filter['category']}|#{filter['name']}|#{filter['value']}"
75
- }.sort.join(',')
130
+ f1_array <=> f2_array
131
+ }
76
132
  end
77
133
 
78
- def evidence_facets(query_hash = nil)
79
- pipeline =
80
- if query_hash.is_a?(Hash) && !query_hash.empty?
81
- [{:'$match' => query_hash}] + AGGREGATION_PIPELINE
82
- else
83
- AGGREGATION_PIPELINE
84
- end
85
- @evidence.aggregate(pipeline)
134
+ def facet_cache_collection(filters)
135
+ result = @evidence_facet_cache.find_one(
136
+ {:filters => filters},
137
+ :fields => {:cache_collection => 1, :_id => 0}
138
+ )
139
+
140
+ result && result['cache_collection']
86
141
  end
87
142
 
88
- AGGREGATION_PIPELINE = [
89
- {
90
- :'$project' => {
91
- :_id => 0,
92
- :facets => 1
93
- }
94
- },
95
- {
96
- :'$unwind' => '$facets'
97
- },
143
+ def create_aggregation(out_collection, match_query = {}, options = {})
144
+ pipeline = CREATE_AGGREGATION[:pipeline] + [{ :$out => out_collection }]
145
+ unless match_query.empty?
146
+ pipeline.unshift({ :$match => match_query })
147
+ end
148
+
98
149
  {
99
- :'$group' => {
100
- :_id => '$facets',
101
- :count => {
102
- :'$sum' => 1
150
+ :pipeline => pipeline,
151
+ :options => CREATE_AGGREGATION[:options].merge(options)
152
+ }
153
+ end
154
+
155
+ # Define the aggregation pipeline
156
+ CREATE_AGGREGATION = {
157
+ :pipeline => [
158
+ {
159
+ :$project => {
160
+ :_id => 0,
161
+ :facets => 1
162
+ }
163
+ },
164
+ {
165
+ :$unwind => '$facets'
166
+ },
167
+ {
168
+ :$group => {
169
+ :_id => '$facets',
170
+ :count => {
171
+ :$sum => 1
172
+ }
173
+ }
174
+ },
175
+ {
176
+ :$sort => {
177
+ :count => -1
178
+ }
179
+ },
180
+ {
181
+ :$group => {
182
+ :_id => {
183
+ :category => '$_id.category',
184
+ :name => '$_id.name'
185
+ },
186
+ :values => {
187
+ :$push => {
188
+ :value => '$_id.value',
189
+ :count => '$count'
190
+ }
191
+ }
192
+ }
193
+ },
194
+ {
195
+ :$project => {
196
+ :category => '$_id.category',
197
+ :name => '$_id.name',
198
+ :values => { :$slice => ['$values', 1000] }
103
199
  }
104
200
  }
105
- },
106
- {
107
- :'$sort' => {
108
- :count => -1
109
- }
201
+ ],
202
+ :options => {
203
+ :allowDiskUse => true
110
204
  }
111
- ]
205
+ }
206
+
207
+ # Define UUID implementation based on Ruby.
208
+ if RUBY_ENGINE =~ /^jruby/i
209
+ java_import 'java.util.UUID'
210
+ define_singleton_method(:generate_uuid) do
211
+ Java::JavaUtil::UUID.random_uuid.to_s
212
+ end
213
+ else
214
+ require 'uuid'
215
+ define_singleton_method(:generate_uuid) do
216
+ UUID.generate
217
+ end
218
+ end
112
219
  end
113
220
  end
114
221
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: openbel-api
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.5.1
4
+ version: 0.6.1
5
5
  platform: java
6
6
  authors:
7
7
  - Anthony Bargnesi
@@ -10,14 +10,14 @@ authors:
10
10
  autorequire:
11
11
  bindir: bin
12
12
  cert_chain: []
13
- date: 2015-12-18 00:00:00.000000000 Z
13
+ date: 2016-03-16 00:00:00.000000000 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
16
  requirement: !ruby/object:Gem::Requirement
17
17
  requirements:
18
18
  - - '='
19
19
  - !ruby/object:Gem::Version
20
- version: 0.4.1
20
+ version: 0.6.0
21
21
  name: bel
22
22
  prerelease: false
23
23
  type: :runtime
@@ -25,7 +25,105 @@ dependencies:
25
25
  requirements:
26
26
  - - '='
27
27
  - !ruby/object:Gem::Version
28
- version: 0.4.1
28
+ version: 0.6.0
29
+ - !ruby/object:Gem::Dependency
30
+ requirement: !ruby/object:Gem::Requirement
31
+ requirements:
32
+ - - '='
33
+ - !ruby/object:Gem::Version
34
+ version: 1.99.0
35
+ name: json-ld
36
+ prerelease: false
37
+ type: :runtime
38
+ version_requirements: !ruby/object:Gem::Requirement
39
+ requirements:
40
+ - - '='
41
+ - !ruby/object:Gem::Version
42
+ version: 1.99.0
43
+ - !ruby/object:Gem::Dependency
44
+ requirement: !ruby/object:Gem::Requirement
45
+ requirements:
46
+ - - '='
47
+ - !ruby/object:Gem::Version
48
+ version: 1.99.0
49
+ name: rdf-json
50
+ prerelease: false
51
+ type: :runtime
52
+ version_requirements: !ruby/object:Gem::Requirement
53
+ requirements:
54
+ - - '='
55
+ - !ruby/object:Gem::Version
56
+ version: 1.99.0
57
+ - !ruby/object:Gem::Dependency
58
+ requirement: !ruby/object:Gem::Requirement
59
+ requirements:
60
+ - - '='
61
+ - !ruby/object:Gem::Version
62
+ version: 1.99.0
63
+ name: rdf-rdfa
64
+ prerelease: false
65
+ type: :runtime
66
+ version_requirements: !ruby/object:Gem::Requirement
67
+ requirements:
68
+ - - '='
69
+ - !ruby/object:Gem::Version
70
+ version: 1.99.0
71
+ - !ruby/object:Gem::Dependency
72
+ requirement: !ruby/object:Gem::Requirement
73
+ requirements:
74
+ - - '='
75
+ - !ruby/object:Gem::Version
76
+ version: 1.99.0
77
+ name: rdf-rdfxml
78
+ prerelease: false
79
+ type: :runtime
80
+ version_requirements: !ruby/object:Gem::Requirement
81
+ requirements:
82
+ - - '='
83
+ - !ruby/object:Gem::Version
84
+ version: 1.99.0
85
+ - !ruby/object:Gem::Dependency
86
+ requirement: !ruby/object:Gem::Requirement
87
+ requirements:
88
+ - - '='
89
+ - !ruby/object:Gem::Version
90
+ version: 1.99.0.1
91
+ name: rdf-trig
92
+ prerelease: false
93
+ type: :runtime
94
+ version_requirements: !ruby/object:Gem::Requirement
95
+ requirements:
96
+ - - '='
97
+ - !ruby/object:Gem::Version
98
+ version: 1.99.0.1
99
+ - !ruby/object:Gem::Dependency
100
+ requirement: !ruby/object:Gem::Requirement
101
+ requirements:
102
+ - - '='
103
+ - !ruby/object:Gem::Version
104
+ version: 1.99.0
105
+ name: rdf-trix
106
+ prerelease: false
107
+ type: :runtime
108
+ version_requirements: !ruby/object:Gem::Requirement
109
+ requirements:
110
+ - - '='
111
+ - !ruby/object:Gem::Version
112
+ version: 1.99.0
113
+ - !ruby/object:Gem::Dependency
114
+ requirement: !ruby/object:Gem::Requirement
115
+ requirements:
116
+ - - '='
117
+ - !ruby/object:Gem::Version
118
+ version: 1.99.0
119
+ name: rdf-turtle
120
+ prerelease: false
121
+ type: :runtime
122
+ version_requirements: !ruby/object:Gem::Requirement
123
+ requirements:
124
+ - - '='
125
+ - !ruby/object:Gem::Version
126
+ version: 1.99.0
29
127
  - !ruby/object:Gem::Dependency
30
128
  requirement: !ruby/object:Gem::Requirement
31
129
  requirements:
@@ -45,7 +143,7 @@ dependencies:
45
143
  requirements:
46
144
  - - '='
47
145
  - !ruby/object:Gem::Version
48
- version: 0.4.1
146
+ version: 0.4.2
49
147
  name: bel-rdf-jena
50
148
  prerelease: false
51
149
  type: :runtime
@@ -53,13 +151,13 @@ dependencies:
53
151
  requirements:
54
152
  - - '='
55
153
  - !ruby/object:Gem::Version
56
- version: 0.4.1
154
+ version: 0.4.2
57
155
  - !ruby/object:Gem::Dependency
58
156
  requirement: !ruby/object:Gem::Requirement
59
157
  requirements:
60
158
  - - '='
61
159
  - !ruby/object:Gem::Version
62
- version: 1.99.0
160
+ version: 1.99.1
63
161
  name: rdf
64
162
  prerelease: false
65
163
  type: :runtime
@@ -67,7 +165,7 @@ dependencies:
67
165
  requirements:
68
166
  - - '='
69
167
  - !ruby/object:Gem::Version
70
- version: 1.99.0
168
+ version: 1.99.1
71
169
  - !ruby/object:Gem::Dependency
72
170
  requirement: !ruby/object:Gem::Requirement
73
171
  requirements:
@@ -171,7 +269,7 @@ dependencies:
171
269
  requirements:
172
270
  - - '='
173
271
  - !ruby/object:Gem::Version
174
- version: 2.15.3
272
+ version: 3.1.0
175
273
  name: puma
176
274
  prerelease: false
177
275
  type: :runtime
@@ -179,7 +277,7 @@ dependencies:
179
277
  requirements:
180
278
  - - '='
181
279
  - !ruby/object:Gem::Version
182
- version: 2.15.3
280
+ version: 3.1.0
183
281
  - !ruby/object:Gem::Dependency
184
282
  requirement: !ruby/object:Gem::Requirement
185
283
  requirements:
@@ -264,20 +362,6 @@ dependencies:
264
362
  - - '='
265
363
  - !ruby/object:Gem::Version
266
364
  version: 1.5.2
267
- - !ruby/object:Gem::Dependency
268
- requirement: !ruby/object:Gem::Requirement
269
- requirements:
270
- - - '='
271
- - !ruby/object:Gem::Version
272
- version: 1.8.0
273
- name: rest-client
274
- prerelease: false
275
- type: :runtime
276
- version_requirements: !ruby/object:Gem::Requirement
277
- requirements:
278
- - - '='
279
- - !ruby/object:Gem::Version
280
- version: 1.8.0
281
365
  description: The OpenBEL API provides a RESTful API over HTTP to manage BEL knowledge.
282
366
  email: abargnesi@selventa.com
283
367
  executables:
@@ -296,7 +380,11 @@ files:
296
380
  - app/openbel/api/app.rb
297
381
  - app/openbel/api/config.rb
298
382
  - app/openbel/api/config.ru
383
+ - app/openbel/api/helpers/base.rb
384
+ - app/openbel/api/helpers/evidence.rb
385
+ - app/openbel/api/helpers/filters.rb
299
386
  - app/openbel/api/helpers/pager.rb
387
+ - app/openbel/api/helpers/translators.rb
300
388
  - app/openbel/api/middleware/auth.rb
301
389
  - app/openbel/api/resources/adapters/basic_json.rb
302
390
  - app/openbel/api/resources/annotation.rb
@@ -333,8 +421,6 @@ files:
333
421
  - app/openbel/api/version.rb
334
422
  - bin/openbel-api
335
423
  - bin/openbel-config
336
- - config/async_evidence.rb
337
- - config/async_jena.rb
338
424
  - config/config.yml
339
425
  - config/server_config.rb
340
426
  - lib/openbel/api/cache/cache.rb