qm 1.2.3 → 1.2.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0aed10f6857450ae540fea58ff7d244cbd25fe2c
-  data.tar.gz: 00aca546287a88a5f110039f63a82f6b53df2e37
+  metadata.gz: 30d506778e43477d1c2ebbf9babd96414aff3eca
+  data.tar.gz: 12f678fe08b9ca4ca9d17eb3da23f4362dc0e63e
 SHA512:
-  metadata.gz: 0372ab2af4f83154ea65e3ca914e1092e49ccedea35da6a375e639a2de6296cd6083253ab6f9a8816a989e8a2b25db9fd069620b9f53d08fa710833795a5aa21
-  data.tar.gz: 66787f708e1cfe4d5354d6df1e2a881350fa8cad4da42f51d02126477bf60bc562e3cc83b5cc4f135975384a241636cc72c2b61b4053bc31de06e86ed922b12a
+  metadata.gz: 191982624850850cc8b0f75abb127bdf60fc77ebfaf07ca02b310674b63863c3d3565c722f40595ac9b253a087b49a2fe9931212803527130d2c2817d87a33d7
+  data.tar.gz: c35d49308fc123a18e23e4eca4e2443756cc26eb2c63e397dc04251bf5eef35646a4ef6ce9346be2d1bcbc7d1eca763e134eb200c3ba9fbb180fdb031c9b71a5

data/lib/qm.rb

@@ -2,46 +2,68 @@
 
 #-  qm.rb ~~
 #                                                       ~~ (c) SRW, 12 Apr 2013
-#                                                   ~~ last updated 23 Jan 2015
+#                                                   ~~ last updated 29 Jan 2015
 
 module QM
 
-    def self::create_app(options = {})
-      # This function creates and configures a fresh Sinatra app that inherits
-      # from the original "teaching version". This code is separated from the
-      # `launch_service` method's code to allow a `QMachineService` instance to
-      # be used from the "config.ru" file of a Rack app.
+    VERSION = '1.2.4'
+
+  module_function
+
+    def create_app(options = {})
+      # This function creates and configures a fresh app. This code is separate
+      # from the `launch_service` method's code to allow direct use of a
+      # `QMachineService` instance from within a Rackup file ("config.ru").
         require 'qm/service'
         app = Sinatra.new(QMachineService) do
             configure do
                 convert = lambda do |x|
                   # This converts all keys in a hash to symbols recursively.
                     if x.is_a?(Hash) then
-                        x = x.inject({}) do |memo, (k, v)|
-                            memo[k.to_sym] = convert.call(v)
-                            memo
+                        x = x.inject({}) do |y, (key, val)|
+                            y[key.to_sym] = convert.call(val); y
                         end
                     end
                     return x
                 end
-                options = convert.call(options)
-                set options
+                convert.call(options).each_pair do |key, val|
+                  # This provides feedback for user-specified options.
+                    if settings.qm_options.include?(key) then
+                        set(key, val)
+                    else
+                        STDERR.puts "Unknown option: #{key}"
+                    end
+                end
+                settings.qm_lazy.each do |key|
+                  # Eagerly evaluate the lambdas in `QMachineService` in the
+                  # correct scope and store their outputs. This strategy avoids
+                  # re-evaluating them for every HTTP request later, of course,
+                  # but the main motivation is to avoid endlessly opening new
+                  # connections without closing old ones.
+                    set(key, settings.send(key))
+                end
             end
         end
         return app
     end
 
-    def self::launch_client(options = {mothership: 'https://api.qmachine.org'})
+    def launch_client(options = {mothership: 'https://api.qmachine.org'})
       # This function needs documentation.
         require 'qm/client'
         return QMachineClient.new(options)
     end
 
-    def self::launch_service(options = {})
+    def launch_service(options = {})
       # This function launches a new app using Unicorn :-)
         require 'unicorn'
-        app = self::create_app(options)
+        app = create_app(options)
         Unicorn::HttpServer.new(app, {
+            before_fork: lambda {|server, worker|
+              # This needs documentation.
+                settings = server.app.settings
+                settings.api_db.close if not settings.api_db.nil?
+                settings.log_db.close if not settings.log_db.nil?
+            },
             listeners: [
                 app.settings.hostname.to_s + ':' + app.settings.port.to_s
             ],
@@ -52,32 +74,11 @@ module QM
         return
     end
 
-=begin
-    def self::launch_service(options = {})
-      # This function launches a new app using Puma. I would prefer to use Puma
-      # instead of Unicorn in the future, in order to support as many of the
-      # different Ruby platforms as possible, but that's not the main priority
-      # for this "teaching version" anyway. Puma will teach *me* a lot about
-      # concurrency in the meantime :-)
-        require 'puma'
-        app = self::create_app(options)
-        server = Puma::Server.new(app)
-        server.add_tcp_listener(app.settings.hostname, app.settings.port)
-        server.min_threads = 1
-        server.max_threads = 16
-        puts "QM up -> http://#{app.settings.hostname}:#{app.settings.port} ..."
-        server.run.join
-        return
-    end
-=end
-
-    def self::version()
+    def version()
       # This function exists because it exists in the Node.js version.
         return VERSION
     end
 
-    VERSION = '1.2.3'
-
 end
 
 #-  vim:set syntax=ruby:

data/lib/qm/defs-mongo.rb

@@ -1,66 +1,83 @@
 #-  Ruby source code
 
 #-  defs-mongo.rb ~~
+#
+#   See http://www.sinatrarb.com/extensions.html.
+#
 #                                                       ~~ (c) SRW, 16 Jul 2014
-#                                                   ~~ last updated 22 Jan 2015
+#                                                   ~~ last updated 29 Jan 2015
 
 require 'json'
 require 'mongo'
-require 'sinatra/base'
 
-module Sinatra
+module QM
 
-    module MongoConnect
+    class MongoStore
 
-        def connect_api_store(opts = settings.persistent_storage)
-          # This helper function needs documentation.
-            db = Mongo::MongoClient.from_uri(opts[:mongo]).db
-            db.collection('avars').ensure_index({
-                box: Mongo::ASCENDING,
-                key: Mongo::ASCENDING
-            }, {
-                unique: true
-            })
-            db.collection('avars').ensure_index('exp_date', {
-                expireAfterSeconds: 0
-            })
-            return db
+        def close()
+          # This function needs documentation.
+            @api_db.connection.close if @api_db.respond_to?('connection')
+            @log_db.connection.close if @log_db.respond_to?('connection')
+            return
         end
 
-        def connect_log_store(opts = settings.trafficlog_storage)
-          # This helper function needs documentation.
-            if opts.has_key?(:mongo) then
-                return Mongo::MongoClient.from_uri(opts[:mongo]).db
+        def connect_api_store(opts = {})
+          # This function needs documentation.
+            if (opts.has_key?(:mongo)) then
+                @api_db ||= Mongo::MongoClient.from_uri(opts[:mongo]).db
+                @api_db.collection('avars').ensure_index({
+                    box: Mongo::ASCENDING,
+                    key: Mongo::ASCENDING
+                }, {
+                    unique: true
+                })
+                @api_db.collection('avars').ensure_index('exp_date', {
+                    expireAfterSeconds: 0
+                })
             end
+            return @api_db
         end
 
-    end
-
-    module MongoAPIDefs
+        def connect_log_store(opts = {})
+          # This function needs documentation.
+            if (opts.has_key?(:mongo)) then
+                @log_db ||= Mongo::MongoClient.from_uri(opts[:mongo]).db
+            end
+            return @log_db
+        end
 
         def get_avar(params)
-          # This helper function needs documentation.
-            x = settings.api_db.collection('avars').find_and_modify({
+          # This helper function retrieves an avar's representation if it
+          # exists, and it also updates the "expiration date" of the avar in
+          # the database so that data still being used for computations will
+          # not be removed.
+            x = @api_db.collection('avars').find_and_modify({
+                fields: {
+                    _id: 0,
+                    body: 1
+                },
                 query: {
                     box: params[0],
                     key: params[1]
                 },
                 update: {
+                  # NOTE: The hash below must use `=>` (not `:`) in JRuby, as
+                  # of version 1.7.18, but QM won't be supporting JRuby anyway
+                  # until (a.) JRuby 9000 is stable and (b.) I understand Puma.
                     '$set': {
-                        exp_date: Time.now + settings.avar_ttl
+                        exp_date: Time.now + @settings.avar_ttl
                     }
                 },
-                fields: {
-                    _id: 0,
-                    body: 1
-                },
                 upsert: false
             })
             return (x.nil?) ? '{}' : x['body']
         end
 
         def get_list(params)
-          # This helper function needs documentation.
+          # This helper function retrieves a list of "key" properties for avars
+          # in the database that have a "status" property, because those are
+          # assumed to represent task descriptions. The function returns the
+          # list as a stringified JSON array in which order is not important.
             opts = {
                 fields: {
                     _id: 0,
@@ -69,59 +86,65 @@ module Sinatra
             }
             query = {
                 box: params[0],
+                exp_date: {
+                    '$gt': Time.now
+                },
                 status: params[1]
             }
             x = []
-            settings.api_db.collection('avars').find(query, opts).each do |doc|
-              # This block needs documentation.
+            @api_db.collection('avars').find(query, opts).each do |doc|
+              # This block appends each task's key to a running list, but the
+              # the order in which the keys are added is *not* sorted.
                 x.push(doc['key'])
             end
             return (x.length == 0) ? '[]' : x.to_json
         end
 
+        def initialize(opts = {})
+          # This constructor function needs documentation.
+            @settings = opts
+        end
+
+        def log(request)
+          # This helper function inserts a new document into MongoDB after each
+          # request. Eventually, this function will be replaced by one that
+          # delegates to a custom `log` function like the Node.js version.
+            @log_db.collection('traffic').insert({
+                host:           request.host,
+                method:         request.request_method,
+                timestamp:      Time.now,
+                url:            request.fullpath
+            }, {
+                w: 0
+            })
+            return
+        end
+
         def set_avar(params)
-          # This helper function needs documentation.
+          # This helper function writes an avar to the database by "upserting"
+          # a Mongo document that represents it.
             doc = {
                 body: params.last,
                 box: params[0],
-                exp_date: Time.now + settings.avar_ttl,
+                exp_date: Time.now + @settings.avar_ttl,
                 key: params[1]
             }
             doc['status'] = params[2] if params.length == 4
             opts = {
                 multi: false,
-                upsert: true
+                upsert: true,
+                w: 1
             }
             query = {
                 box: params[0],
                 key: params[1]
             }
-            settings.api_db.collection('avars').update(query, doc, opts)
-            return
-        end
-
-    end
-
-    module MongoLogDefs
-
-        def log_to_db()
-          # This helper function needs documentation.
-            settings.log_db.collection('traffic').insert({
-                host:           request.host,
-                ip:             request.ip,
-                method:         request.request_method,
-                status_code:    response.status,
-                timestamp:      Time.now,
-                url:            request.fullpath
-            })
+            @api_db.collection('avars').update(query, doc, opts)
             return
         end
 
     end
 
-    helpers MongoAPIDefs, MongoLogDefs
-    register MongoConnect
-
 end
 
 #-  vim:set syntax=ruby:

data/lib/qm/service.rb

@@ -16,26 +16,36 @@
 #   of a 'box', 'key', or 'status' value.
 #
 #                                                       ~~ (c) SRW, 24 Apr 2013
-#                                                   ~~ last updated 23 Jan 2015
+#                                                   ~~ last updated 29 Jan 2015
 
-require 'qm/defs-mongo'
+require 'qm/storage'
 require 'sinatra/base'
 require 'sinatra/cross_origin'
 
 class QMachineService < Sinatra::Base
 
-    register Sinatra::CrossOrigin
-    register Sinatra::MongoConnect
+    register Sinatra::CrossOrigin, QM::StorageConnector
 
     configure do
 
-      # Helper methods
+      # MIME types that may be missing
 
-        helpers Sinatra::MongoAPIDefs, Sinatra::MongoLogDefs
+        mime_type({
+            webapp:             'application/x-web-app-manifest+json'
+        })
 
-      # QMachine options
+      # QMachine definitions
 
-        set avar_ttl:           86400, # seconds (24 * 60 * 60 = 1 day)
+        qm_lazy = {
+            api_db:             lambda { connect_api_store },
+            bind:               lambda { settings.hostname },
+            log_db:             lambda { connect_log_store },
+            logging:            lambda { settings.log_db.nil? },
+            static:             lambda { settings.enable_web_server }
+        }
+
+        qm_options = {
+            avar_ttl:           86400, # seconds (24 * 60 * 60 = 1 day)
             enable_api_server:  false,
             enable_cors:        false,
             enable_web_server:  false,
@@ -46,22 +56,25 @@ class QMachineService < Sinatra::Base
             public_folder:      'public',
             trafficlog_storage: {},
             worker_procs:       1
+        }
 
-      # Sinatra mappings and options needed by QMachine
+      # Save the configuration :-)
 
-        mime_type webapp:       'application/x-web-app-manifest+json'
+        set(qm_lazy.merge(qm_options).merge({
+
+          # The first two entries here are needed by `QM.create_app`.
+
+            qm_lazy:            qm_lazy.keys,
+            qm_options:         qm_options.keys,
+
+          # The rest are Rack / Sinatra mappings.
 
-        set api_db:             lambda { connect_api_store },
-            bind:               lambda { settings.hostname },
-            log_db:             lambda { connect_log_store },
-            logging:            lambda { settings.log_db.nil? },
             raise_errors:       false,
             run:                false,
             show_exceptions:    false,
-            static:             lambda { settings.enable_web_server },
             x_cascade:          false
 
-      # See also: http://www.sinatrarb.com/configuration.html
+        }))
 
     end
 
@@ -88,6 +101,11 @@ class QMachineService < Sinatra::Base
             halt [444, headers, ['']]
         end
 
+        def log_to_db()
+          # This helper function needs documentation.
+            settings.log_db.log(request)
+        end
+
     end
 
     not_found do
@@ -98,7 +116,9 @@ class QMachineService < Sinatra::Base
   # Filter definitions
 
     after do
-        log_to_db unless response.status === 444 or settings.logging == true
+      # After every successful request, if logging to stdout has been disabled,
+      # write a new entry into the traffic log database.
+        log_to_db unless response.status == 444 or settings.logging == true
     end
 
     before '/:version/:box' do
@@ -122,10 +142,10 @@ class QMachineService < Sinatra::Base
       # such as when checking for new tasks to run or downloading results.
         if @key.is_a?(String) then
           # This arm runs when a client requests the value of a specific avar.
-            y = get_avar([@box, @key])
+            y = settings.api_db.get_avar([@box, @key])
         else
           # This arm runs when a client requests a task queue.
-            y = get_list([@box, @status])
+            y = settings.api_db.get_list([@box, @status])
         end
         return [200, {'Content-Type' => 'application/json'}, [y]]
     end
@@ -145,10 +165,10 @@ class QMachineService < Sinatra::Base
           # This arm runs only when a client writes an avar which represents a
           # task description.
             hang_up unless x['status'].match(/^[\w\-]+$/)
-            set_avar([@box, @key, x['status'], body])
+            settings.api_db.set_avar([@box, @key, x['status'], body])
         else
           # This arm runs when a client is writing a "regular avar".
-            set_avar([@box, @key, body])
+            settings.api_db.set_avar([@box, @key, body])
         end
         return [201, {'Content-Type' => 'text/plain'}, ['']]
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: qm
 version: !ruby/object:Gem::Version
-  version: 1.2.3
+  version: 1.2.4
 platform: ruby
 authors:
 - Sean Wilkinson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-01-23 00:00:00.000000000 Z
+date: 2015-01-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bson_ext
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.11.1
+        version: 1.12.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.11.1
+        version: 1.12.0
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement
@@ -58,14 +58,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.11.1
+        version: 1.12.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.11.1
+        version: 1.12.0
 - !ruby/object:Gem::Dependency
   name: sinatra
   requirement: !ruby/object:Gem::Requirement