qm 1.2.3 → 1.2.4

Sign up to get free protection for your applications and to get access to all the features.
Files changed (5) hide show
  1. checksums.yaml +4 -4
  2. data/lib/qm.rb +37 -36
  3. data/lib/qm/defs-mongo.rb +81 -58
  4. data/lib/qm/service.rb +41 -21
  5. metadata +6 -6
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 0aed10f6857450ae540fea58ff7d244cbd25fe2c
4
- data.tar.gz: 00aca546287a88a5f110039f63a82f6b53df2e37
3
+ metadata.gz: 30d506778e43477d1c2ebbf9babd96414aff3eca
4
+ data.tar.gz: 12f678fe08b9ca4ca9d17eb3da23f4362dc0e63e
5
5
  SHA512:
6
- metadata.gz: 0372ab2af4f83154ea65e3ca914e1092e49ccedea35da6a375e639a2de6296cd6083253ab6f9a8816a989e8a2b25db9fd069620b9f53d08fa710833795a5aa21
7
- data.tar.gz: 66787f708e1cfe4d5354d6df1e2a881350fa8cad4da42f51d02126477bf60bc562e3cc83b5cc4f135975384a241636cc72c2b61b4053bc31de06e86ed922b12a
6
+ metadata.gz: 191982624850850cc8b0f75abb127bdf60fc77ebfaf07ca02b310674b63863c3d3565c722f40595ac9b253a087b49a2fe9931212803527130d2c2817d87a33d7
7
+ data.tar.gz: c35d49308fc123a18e23e4eca4e2443756cc26eb2c63e397dc04251bf5eef35646a4ef6ce9346be2d1bcbc7d1eca763e134eb200c3ba9fbb180fdb031c9b71a5
data/lib/qm.rb CHANGED
@@ -2,46 +2,68 @@
2
2
 
3
3
  #- qm.rb ~~
4
4
  # ~~ (c) SRW, 12 Apr 2013
5
- # ~~ last updated 23 Jan 2015
5
+ # ~~ last updated 29 Jan 2015
6
6
 
7
7
  module QM
8
8
 
9
- def self::create_app(options = {})
10
- # This function creates and configures a fresh Sinatra app that inherits
11
- # from the original "teaching version". This code is separated from the
12
- # `launch_service` method's code to allow a `QMachineService` instance to
13
- # be used from the "config.ru" file of a Rack app.
9
+ VERSION = '1.2.4'
10
+
11
+ module_function
12
+
13
+ def create_app(options = {})
14
+ # This function creates and configures a fresh app. This code is separate
15
+ # from the `launch_service` method's code to allow direct use of a
16
+ # `QMachineService` instance from within a Rackup file ("config.ru").
14
17
  require 'qm/service'
15
18
  app = Sinatra.new(QMachineService) do
16
19
  configure do
17
20
  convert = lambda do |x|
18
21
  # This converts all keys in a hash to symbols recursively.
19
22
  if x.is_a?(Hash) then
20
- x = x.inject({}) do |memo, (k, v)|
21
- memo[k.to_sym] = convert.call(v)
22
- memo
23
+ x = x.inject({}) do |y, (key, val)|
24
+ y[key.to_sym] = convert.call(val); y
23
25
  end
24
26
  end
25
27
  return x
26
28
  end
27
- options = convert.call(options)
28
- set options
29
+ convert.call(options).each_pair do |key, val|
30
+ # This provides feedback for user-specified options.
31
+ if settings.qm_options.include?(key) then
32
+ set(key, val)
33
+ else
34
+ STDERR.puts "Unknown option: #{key}"
35
+ end
36
+ end
37
+ settings.qm_lazy.each do |key|
38
+ # Eagerly evaluate the lambdas in `QMachineService` in the
39
+ # correct scope and store their outputs. This strategy avoids
40
+ # re-evaluating them for every HTTP request later, of course,
41
+ # but the main motivation is to avoid endlessly opening new
42
+ # connections without closing old ones.
43
+ set(key, settings.send(key))
44
+ end
29
45
  end
30
46
  end
31
47
  return app
32
48
  end
33
49
 
34
- def self::launch_client(options = {mothership: 'https://api.qmachine.org'})
50
+ def launch_client(options = {mothership: 'https://api.qmachine.org'})
35
51
  # This function needs documentation.
36
52
  require 'qm/client'
37
53
  return QMachineClient.new(options)
38
54
  end
39
55
 
40
- def self::launch_service(options = {})
56
+ def launch_service(options = {})
41
57
  # This function launches a new app using Unicorn :-)
42
58
  require 'unicorn'
43
- app = self::create_app(options)
59
+ app = create_app(options)
44
60
  Unicorn::HttpServer.new(app, {
61
+ before_fork: lambda {|server, worker|
62
+ # This needs documentation.
63
+ settings = server.app.settings
64
+ settings.api_db.close if not settings.api_db.nil?
65
+ settings.log_db.close if not settings.log_db.nil?
66
+ },
45
67
  listeners: [
46
68
  app.settings.hostname.to_s + ':' + app.settings.port.to_s
47
69
  ],
@@ -52,32 +74,11 @@ module QM
52
74
  return
53
75
  end
54
76
 
55
- =begin
56
- def self::launch_service(options = {})
57
- # This function launches a new app using Puma. I would prefer to use Puma
58
- # instead of Unicorn in the future, in order to support as many of the
59
- # different Ruby platforms as possible, but that's not the main priority
60
- # for this "teaching version" anyway. Puma will teach *me* a lot about
61
- # concurrency in the meantime :-)
62
- require 'puma'
63
- app = self::create_app(options)
64
- server = Puma::Server.new(app)
65
- server.add_tcp_listener(app.settings.hostname, app.settings.port)
66
- server.min_threads = 1
67
- server.max_threads = 16
68
- puts "QM up -> http://#{app.settings.hostname}:#{app.settings.port} ..."
69
- server.run.join
70
- return
71
- end
72
- =end
73
-
74
- def self::version()
77
+ def version()
75
78
  # This function exists because it exists in the Node.js version.
76
79
  return VERSION
77
80
  end
78
81
 
79
- VERSION = '1.2.3'
80
-
81
82
  end
82
83
 
83
84
  #- vim:set syntax=ruby:
data/lib/qm/defs-mongo.rb CHANGED
@@ -1,66 +1,83 @@
1
1
  #- Ruby source code
2
2
 
3
3
  #- defs-mongo.rb ~~
4
+ #
5
+ # See http://www.sinatrarb.com/extensions.html.
6
+ #
4
7
  # ~~ (c) SRW, 16 Jul 2014
5
- # ~~ last updated 22 Jan 2015
8
+ # ~~ last updated 29 Jan 2015
6
9
 
7
10
  require 'json'
8
11
  require 'mongo'
9
- require 'sinatra/base'
10
12
 
11
- module Sinatra
13
+ module QM
12
14
 
13
- module MongoConnect
15
+ class MongoStore
14
16
 
15
- def connect_api_store(opts = settings.persistent_storage)
16
- # This helper function needs documentation.
17
- db = Mongo::MongoClient.from_uri(opts[:mongo]).db
18
- db.collection('avars').ensure_index({
19
- box: Mongo::ASCENDING,
20
- key: Mongo::ASCENDING
21
- }, {
22
- unique: true
23
- })
24
- db.collection('avars').ensure_index('exp_date', {
25
- expireAfterSeconds: 0
26
- })
27
- return db
17
+ def close()
18
+ # This function needs documentation.
19
+ @api_db.connection.close if @api_db.respond_to?('connection')
20
+ @log_db.connection.close if @log_db.respond_to?('connection')
21
+ return
28
22
  end
29
23
 
30
- def connect_log_store(opts = settings.trafficlog_storage)
31
- # This helper function needs documentation.
32
- if opts.has_key?(:mongo) then
33
- return Mongo::MongoClient.from_uri(opts[:mongo]).db
24
+ def connect_api_store(opts = {})
25
+ # This function needs documentation.
26
+ if (opts.has_key?(:mongo)) then
27
+ @api_db ||= Mongo::MongoClient.from_uri(opts[:mongo]).db
28
+ @api_db.collection('avars').ensure_index({
29
+ box: Mongo::ASCENDING,
30
+ key: Mongo::ASCENDING
31
+ }, {
32
+ unique: true
33
+ })
34
+ @api_db.collection('avars').ensure_index('exp_date', {
35
+ expireAfterSeconds: 0
36
+ })
34
37
  end
38
+ return @api_db
35
39
  end
36
40
 
37
- end
38
-
39
- module MongoAPIDefs
41
+ def connect_log_store(opts = {})
42
+ # This function needs documentation.
43
+ if (opts.has_key?(:mongo)) then
44
+ @log_db ||= Mongo::MongoClient.from_uri(opts[:mongo]).db
45
+ end
46
+ return @log_db
47
+ end
40
48
 
41
49
  def get_avar(params)
42
- # This helper function needs documentation.
43
- x = settings.api_db.collection('avars').find_and_modify({
50
+ # This helper function retrieves an avar's representation if it
51
+ # exists, and it also updates the "expiration date" of the avar in
52
+ # the database so that data still being used for computations will
53
+ # not be removed.
54
+ x = @api_db.collection('avars').find_and_modify({
55
+ fields: {
56
+ _id: 0,
57
+ body: 1
58
+ },
44
59
  query: {
45
60
  box: params[0],
46
61
  key: params[1]
47
62
  },
48
63
  update: {
64
+ # NOTE: The hash below must use `=>` (not `:`) in JRuby, as
65
+ # of version 1.7.18, but QM won't be supporting JRuby anyway
66
+ # until (a.) JRuby 9000 is stable and (b.) I understand Puma.
49
67
  '$set': {
50
- exp_date: Time.now + settings.avar_ttl
68
+ exp_date: Time.now + @settings.avar_ttl
51
69
  }
52
70
  },
53
- fields: {
54
- _id: 0,
55
- body: 1
56
- },
57
71
  upsert: false
58
72
  })
59
73
  return (x.nil?) ? '{}' : x['body']
60
74
  end
61
75
 
62
76
  def get_list(params)
63
- # This helper function needs documentation.
77
+ # This helper function retrieves a list of "key" properties for avars
78
+ # in the database that have a "status" property, because those are
79
+ # assumed to represent task descriptions. The function returns the
80
+ # list as a stringified JSON array in which order is not important.
64
81
  opts = {
65
82
  fields: {
66
83
  _id: 0,
@@ -69,59 +86,65 @@ module Sinatra
69
86
  }
70
87
  query = {
71
88
  box: params[0],
89
+ exp_date: {
90
+ '$gt': Time.now
91
+ },
72
92
  status: params[1]
73
93
  }
74
94
  x = []
75
- settings.api_db.collection('avars').find(query, opts).each do |doc|
76
- # This block needs documentation.
95
+ @api_db.collection('avars').find(query, opts).each do |doc|
96
+ # This block appends each task's key to a running list, but the
97
+ # the order in which the keys are added is *not* sorted.
77
98
  x.push(doc['key'])
78
99
  end
79
100
  return (x.length == 0) ? '[]' : x.to_json
80
101
  end
81
102
 
103
+ def initialize(opts = {})
104
+ # This constructor function needs documentation.
105
+ @settings = opts
106
+ end
107
+
108
+ def log(request)
109
+ # This helper function inserts a new document into MongoDB after each
110
+ # request. Eventually, this function will be replaced by one that
111
+ # delegates to a custom `log` function like the Node.js version.
112
+ @log_db.collection('traffic').insert({
113
+ host: request.host,
114
+ method: request.request_method,
115
+ timestamp: Time.now,
116
+ url: request.fullpath
117
+ }, {
118
+ w: 0
119
+ })
120
+ return
121
+ end
122
+
82
123
  def set_avar(params)
83
- # This helper function needs documentation.
124
+ # This helper function writes an avar to the database by "upserting"
125
+ # a Mongo document that represents it.
84
126
  doc = {
85
127
  body: params.last,
86
128
  box: params[0],
87
- exp_date: Time.now + settings.avar_ttl,
129
+ exp_date: Time.now + @settings.avar_ttl,
88
130
  key: params[1]
89
131
  }
90
132
  doc['status'] = params[2] if params.length == 4
91
133
  opts = {
92
134
  multi: false,
93
- upsert: true
135
+ upsert: true,
136
+ w: 1
94
137
  }
95
138
  query = {
96
139
  box: params[0],
97
140
  key: params[1]
98
141
  }
99
- settings.api_db.collection('avars').update(query, doc, opts)
100
- return
101
- end
102
-
103
- end
104
-
105
- module MongoLogDefs
106
-
107
- def log_to_db()
108
- # This helper function needs documentation.
109
- settings.log_db.collection('traffic').insert({
110
- host: request.host,
111
- ip: request.ip,
112
- method: request.request_method,
113
- status_code: response.status,
114
- timestamp: Time.now,
115
- url: request.fullpath
116
- })
142
+ @api_db.collection('avars').update(query, doc, opts)
117
143
  return
118
144
  end
119
145
 
120
146
  end
121
147
 
122
- helpers MongoAPIDefs, MongoLogDefs
123
- register MongoConnect
124
-
125
148
  end
126
149
 
127
150
  #- vim:set syntax=ruby:
data/lib/qm/service.rb CHANGED
@@ -16,26 +16,36 @@
16
16
  # of a 'box', 'key', or 'status' value.
17
17
  #
18
18
  # ~~ (c) SRW, 24 Apr 2013
19
- # ~~ last updated 23 Jan 2015
19
+ # ~~ last updated 29 Jan 2015
20
20
 
21
- require 'qm/defs-mongo'
21
+ require 'qm/storage'
22
22
  require 'sinatra/base'
23
23
  require 'sinatra/cross_origin'
24
24
 
25
25
  class QMachineService < Sinatra::Base
26
26
 
27
- register Sinatra::CrossOrigin
28
- register Sinatra::MongoConnect
27
+ register Sinatra::CrossOrigin, QM::StorageConnector
29
28
 
30
29
  configure do
31
30
 
32
- # Helper methods
31
+ # MIME types that may be missing
33
32
 
34
- helpers Sinatra::MongoAPIDefs, Sinatra::MongoLogDefs
33
+ mime_type({
34
+ webapp: 'application/x-web-app-manifest+json'
35
+ })
35
36
 
36
- # QMachine options
37
+ # QMachine definitions
37
38
 
38
- set avar_ttl: 86400, # seconds (24 * 60 * 60 = 1 day)
39
+ qm_lazy = {
40
+ api_db: lambda { connect_api_store },
41
+ bind: lambda { settings.hostname },
42
+ log_db: lambda { connect_log_store },
43
+ logging: lambda { settings.log_db.nil? },
44
+ static: lambda { settings.enable_web_server }
45
+ }
46
+
47
+ qm_options = {
48
+ avar_ttl: 86400, # seconds (24 * 60 * 60 = 1 day)
39
49
  enable_api_server: false,
40
50
  enable_cors: false,
41
51
  enable_web_server: false,
@@ -46,22 +56,25 @@ class QMachineService < Sinatra::Base
46
56
  public_folder: 'public',
47
57
  trafficlog_storage: {},
48
58
  worker_procs: 1
59
+ }
49
60
 
50
- # Sinatra mappings and options needed by QMachine
61
+ # Save the configuration :-)
51
62
 
52
- mime_type webapp: 'application/x-web-app-manifest+json'
63
+ set(qm_lazy.merge(qm_options).merge({
64
+
65
+ # The first two entries here are needed by `QM.create_app`.
66
+
67
+ qm_lazy: qm_lazy.keys,
68
+ qm_options: qm_options.keys,
69
+
70
+ # The rest are Rack / Sinatra mappings.
53
71
 
54
- set api_db: lambda { connect_api_store },
55
- bind: lambda { settings.hostname },
56
- log_db: lambda { connect_log_store },
57
- logging: lambda { settings.log_db.nil? },
58
72
  raise_errors: false,
59
73
  run: false,
60
74
  show_exceptions: false,
61
- static: lambda { settings.enable_web_server },
62
75
  x_cascade: false
63
76
 
64
- # See also: http://www.sinatrarb.com/configuration.html
77
+ }))
65
78
 
66
79
  end
67
80
 
@@ -88,6 +101,11 @@ class QMachineService < Sinatra::Base
88
101
  halt [444, headers, ['']]
89
102
  end
90
103
 
104
+ def log_to_db()
105
+ # This helper function needs documentation.
106
+ settings.log_db.log(request)
107
+ end
108
+
91
109
  end
92
110
 
93
111
  not_found do
@@ -98,7 +116,9 @@ class QMachineService < Sinatra::Base
98
116
  # Filter definitions
99
117
 
100
118
  after do
101
- log_to_db unless response.status === 444 or settings.logging == true
119
+ # After every successful request, if logging to stdout has been disabled,
120
+ # write a new entry into the traffic log database.
121
+ log_to_db unless response.status == 444 or settings.logging == true
102
122
  end
103
123
 
104
124
  before '/:version/:box' do
@@ -122,10 +142,10 @@ class QMachineService < Sinatra::Base
122
142
  # such as when checking for new tasks to run or downloading results.
123
143
  if @key.is_a?(String) then
124
144
  # This arm runs when a client requests the value of a specific avar.
125
- y = get_avar([@box, @key])
145
+ y = settings.api_db.get_avar([@box, @key])
126
146
  else
127
147
  # This arm runs when a client requests a task queue.
128
- y = get_list([@box, @status])
148
+ y = settings.api_db.get_list([@box, @status])
129
149
  end
130
150
  return [200, {'Content-Type' => 'application/json'}, [y]]
131
151
  end
@@ -145,10 +165,10 @@ class QMachineService < Sinatra::Base
145
165
  # This arm runs only when a client writes an avar which represents a
146
166
  # task description.
147
167
  hang_up unless x['status'].match(/^[\w\-]+$/)
148
- set_avar([@box, @key, x['status'], body])
168
+ settings.api_db.set_avar([@box, @key, x['status'], body])
149
169
  else
150
170
  # This arm runs when a client is writing a "regular avar".
151
- set_avar([@box, @key, body])
171
+ settings.api_db.set_avar([@box, @key, body])
152
172
  end
153
173
  return [201, {'Content-Type' => 'text/plain'}, ['']]
154
174
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: qm
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.2.3
4
+ version: 1.2.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - Sean Wilkinson
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-01-23 00:00:00.000000000 Z
11
+ date: 2015-01-29 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bson_ext
@@ -16,14 +16,14 @@ dependencies:
16
16
  requirements:
17
17
  - - '='
18
18
  - !ruby/object:Gem::Version
19
- version: 1.11.1
19
+ version: 1.12.0
20
20
  type: :runtime
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
24
  - - '='
25
25
  - !ruby/object:Gem::Version
26
- version: 1.11.1
26
+ version: 1.12.0
27
27
  - !ruby/object:Gem::Dependency
28
28
  name: json
29
29
  requirement: !ruby/object:Gem::Requirement
@@ -58,14 +58,14 @@ dependencies:
58
58
  requirements:
59
59
  - - '='
60
60
  - !ruby/object:Gem::Version
61
- version: 1.11.1
61
+ version: 1.12.0
62
62
  type: :runtime
63
63
  prerelease: false
64
64
  version_requirements: !ruby/object:Gem::Requirement
65
65
  requirements:
66
66
  - - '='
67
67
  - !ruby/object:Gem::Version
68
- version: 1.11.1
68
+ version: 1.12.0
69
69
  - !ruby/object:Gem::Dependency
70
70
  name: sinatra
71
71
  requirement: !ruby/object:Gem::Requirement