mongoscript 0.0.8

data/.gitignore

@@ -0,0 +1,19 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+
+.rvmrc

data/Gemfile

@@ -0,0 +1,32 @@
+source 'http://rubygems.org'
+
+# Specify your gem's dependencies in mongoscript.gemspec
+gemspec
+
+group :development do
+  gem "yard"
+end
+
+group :development, :test do
+  # ORM
+  gem "mongoid", "~> 2.2"
+  gem "bson_ext"
+
+  # Testing infrastructure
+  gem 'rspec'
+  gem 'mocha'
+  gem 'guard'
+  gem 'guard-rspec'
+  gem "parallel_tests"
+  gem "fuubar"
+  gem "rake"
+
+  # testing bundled Javascripts
+  gem "jasmine"
+
+  if RUBY_PLATFORM =~ /darwin/
+    # OS X integration
+    gem "ruby_gntp"
+    gem "rb-fsevent", "~> 0.4.3.1"
+  end
+end

data/Guardfile

@@ -0,0 +1,19 @@
+# A sample Guardfile
+# More info at https://github.com/guard/guard#readme
+
+guard 'rspec', :version => 2, :cli => "--colour --format Fuubar" do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/(.+)\.rb$})     { |m| "spec/lib/#{m[1]}_spec.rb" }
+  watch('spec/spec_helper.rb')  { "spec" }
+
+  # Rails example
+  watch(%r{^app/(.+)\.rb$})                           { |m| "spec/#{m[1]}_spec.rb" }
+  watch(%r{^app/(.*)(\.erb|\.haml)$})                 { |m| "spec/#{m[1]}#{m[2]}_spec.rb" }
+  watch(%r{^app/controllers/(.+)_(controller)\.rb$})  { |m| ["spec/routing/#{m[1]}_routing_spec.rb", "spec/#{m[2]}s/#{m[1]}_#{m[2]}_spec.rb", "spec/acceptance/#{m[1]}_spec.rb"] }
+  watch(%r{^spec/support/(.+)\.rb$})                  { "spec" }
+  watch('config/routes.rb')                           { "spec/routing" }
+  watch('app/controllers/application_controller.rb')  { "spec/controllers" }
+  # Capybara request specs
+  watch(%r{^app/views/(.+)/.*\.(erb|haml)$})          { |m| "spec/requests/#{m[1]}_spec.rb" }
+end
+

data/Rakefile

@@ -0,0 +1,11 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+
+begin
+  require 'jasmine'
+  load 'jasmine/tasks/jasmine.rake'
+rescue LoadError
+  task :jasmine do
+    abort "Jasmine is not available. In order to run jasmine, you must: (sudo) gem install jasmine"
+  end
+end

data/lib/mongoscript/execution.rb

@@ -0,0 +1,90 @@
+require 'digest/md5'
+
+module MongoScript
+  # Execute raw code on the Mongo server
+  # code_for can
+  # longer term, we could store functions on the server
+  # and have deploy tasks that delete all and reinstall stored method on deploys
+  # see http://neovintage.blogspot.com/2010/07/mongodb-stored-javascript-functions.html
+  # 10gen recommends not using stored functions, but that's mainly b/c
+  # they should be stored and versioned with other code
+  # but a cool 6W gem for automatically clearing and installing such code would meet that objection
+  # see http://www.mongodb.org/display/DOCS/Server-side+Code+Execution#Server-sideCodeExecution-Storingfunctionsserverside
+
+  module Execution
+
+    LOADED_SCRIPTS = {}
+
+    class ScriptNotFound < Errno::ENOENT; end
+    class NoScriptDirectory < ArgumentError; end
+    class ExecutionFailure < RuntimeError; end
+
+    def self.included(base)
+      base.class_eval do
+        class << self
+          attr_accessor :script_dirs
+        end
+
+        def self.gem_path
+          mongoscript = Bundler.load.specs.find {|s| s.name == "mongoscript"}
+          File.join(mongoscript.full_gem_path, "lib", "mongoscript", "javascripts")
+        end
+
+        # start out with the scripts provided by the gem
+        @script_dirs = [gem_path]
+
+        extend MongoScript::Execution::ClassMethods
+      end
+    end
+
+    module ClassMethods
+      # code from stored files
+      def code_for(script_name)
+        script_name = script_name.to_s
+        dir = @script_dirs.find {|d| File.exists?(File.join(d, "#{script_name}.js"))}
+        raise ScriptNotFound, "Unable to find script #{script_name}" unless dir
+        code = File.read(File.join(dir, "#{script_name}.js"))
+        LOADED_SCRIPTS[script_name] ||= code
+
+        # for future extension
+        # code_hash = Digest::MD5.hex_digest(code)
+        # LOADED_SCRIPTS[script_name] = {
+        #   :code => code,
+        #   :hash => code_hash,
+        #   :installed => false
+        # }
+        # code
+      end
+
+      def execute_readonly_routine(script_name, *args)
+        execute_readonly_code(code_for(script_name), *args)
+      end
+
+      def execute_readwrite_routine(script_name, *args)
+        execute_readwrite_code(code_for(script_name), *args)
+      end
+
+      # raw code
+      # note: to pass in Mongo options for the $exec call, like nolock, you need to call execute directly
+      # since otherwise we have no way to tell Mongo options from an argument list ending in a JS hash
+      def execute_readonly_code(code, *args)
+        # for readonly operations, set nolock to true to improve concurrency
+        # http://www.mongodb.org/display/DOCS/Server-side+Code+Execution#Server-sideCodeExecution-NotesonConcurrency
+        execute(code, args, {:nolock => true})
+      end
+
+      def execute_readwrite_code(code, *args)
+        execute(code, args)
+      end
+
+      def execute(code, args = [], options = {})
+        # see http://mrdanadams.com/2011/mongodb-eval-ruby-driver/
+        result = MongoScript.database.command({:$eval => code, args: resolve_arguments(args)}.merge(options))
+        unless result["ok"] == 1.0
+          raise ExecutionFailure, "MongoScript.execute JS didn't return {ok: 1.0}!  Result: #{result.inspect}"
+        end
+        result["retval"]
+      end
+    end
+  end
+end

data/lib/mongoscript/javascripts/multiquery.js

@@ -0,0 +1,40 @@
+function multiquery(queries) {
+  var results = {}, collectionObject;
+
+  // extract all the collection names,
+  // so we can return an error if one is provided that doesn't exist
+  // since db[invalidName] returns a collection object
+  // we could perhaps make this more optimized by building a hash,
+  // but it shouldn't be a problem in most cases
+  var collectionNames = db["system.namespaces"].find().map(function(ns) { return ns.name }),
+      dbName = db.toString(),
+      query, base, modifiers, collection;
+
+  for (var queryName in queries) {
+    try {
+      query = queries[queryName];
+      modifiers = query.modifiers || [];
+      collection = query.collection;
+
+      if (collectionNames.indexOf(dbName + "." + collection) !== -1) {
+        base = db[collection].find(query.selector, query.fields || null)
+        // apply any number of modifiers, such as sort, limit, etc.
+        for (var modifier in modifiers) {
+          base = base[modifier](modifiers[modifier])
+        }
+        results[queryName] = base.toArray();
+      }
+      else {
+        // if the collection doesn't exist, return an error
+        // (rather than null -- the DB allows queries on non-existent collections)
+        // doing this here saves us a database call in Ruby
+        results[queryName] = {error: "Unable to locate collection " + (collection ? collection.toString() : collection)}
+      }
+    }
+    catch(e) {
+      results[queryName] = {error: e};
+    }
+  }
+
+  return results;
+}

data/lib/mongoscript/multiquery.rb

@@ -0,0 +1,171 @@
+module MongoScript
+  module Multiquery
+
+    class QueryFailedError < RuntimeError
+      # The original query whose execution failed.
+      attr_accessor :query_parameters
+      # The name of the original query
+      attr_accessor :query_name
+      # The response from the multiquery Javascript.
+      attr_accessor :db_response
+
+      def initialize(name, query, response)
+        @query_name = name
+        @query_parameters = query
+        @db_response = response
+        super("Query #{@query_name} failed with the following response: #{response.inspect}")
+        # set the backtrace to everything that's going on except this initialize method
+        set_backtrace(caller[1...caller.length])
+      end
+    end
+
+    # @private
+    def self.included(base)
+      base.class_eval do
+        extend MongoScript::Multiquery::ClassMethods
+      end
+    end
+
+    module ClassMethods
+
+      # Runs multiple find queries at once,
+      # returning the results keyed to the original query names.
+      # If a query produces an error, its result will be a QueryFailedError object
+      # with the appropriate details; other queries will be unaffected.
+      #
+      # @example
+      #   MongoScript.multiquery({
+      #     # simplest form -- the name is used as the collection to be queried
+      #     :cars => {:query => {:_id => {$in: my_ids}}},
+      #     # the name can also be arbitrary if you explicitly specify a collection
+      #     # (allowing you to query the same collection twice)
+      #     :my_cool_query => {:collection => :books, :objects }
+      #     # you can also pass in Mongoid criteria
+      #     :planes => Plane.where(manufacturer: "Boeing").sort("created_at desc").only(:_id),
+      #   })
+      #   => {
+      #     # results get automatically turned into
+      #     :cars => [#<Car: details>, #<Car: details>],
+      #     :my_cool_query => [#<Book: details>],
+      #     :planes => #<QueryFailedError>
+      #   }
+      #
+      # @raises ArgumentError if the input isn't valid (see #normalize_queries) (see #validate_queries!)
+      #
+      # @returns Hash a set of database results/errors for each query
+      def multiquery(queries)
+        # don't do anything if we don't get any queries
+        return {} if queries == {}
+
+        # resolve all the
+        queries = normalize_queries(queries)
+        validate_queries!(queries)
+        results = MongoScript.execute_readonly_routine("multiquery", mongoize_queries(queries))
+        process_results(results, queries)
+      end
+
+      # Standardize a set of queries into a form we can use.
+      # Specifically, ensure each query has a database collection
+      # and an ORM class, and resolve any ORM criteria into hashes.
+      #
+      # @param queries a set of query_name => hash_or_orm_criteria pairs
+      #
+      # @raises ArgumentError if the query details can't be processed (aren't a hash or Mongoid::Criteria)
+      #
+      # @returns [Hash] a set of hashes that can be fed to mongoize_queries
+      #                 and later used for processing
+      def normalize_queries(queries)
+        # need to also ensure we have details[:klass]
+        queries.inject({}) do |normalized_queries, data|
+          name, details = data
+
+          if details.is_a?(Hash)
+            # duplicate the details so we don't make changes to the original query data
+            details = details.dup.with_indifferent_access
+
+            # if no collection is specified, assume it's the same as the name
+            details[:collection] ||= name
+
+            # ensure that we know which class the collection maps to
+            # so we can rehydrate the resulting data
+            unless details[:klass]
+              expected_class_name = details[:collection].to_s.singularize.titlecase
+              # if the class doesn't exist, this'll be false (and we'll raise an error later)
+              details[:klass] = Object.const_defined?(expected_class_name) && Object.const_get(expected_class_name)
+            end
+          elsif processable_into_parameters?(details)
+            # process Mongo ORM selectors into JS-compatible hashes
+            details = MongoScript.build_multiquery_parameters(details)
+          else
+            raise ArgumentError, "Invalid selector type provided to multiquery for #{name}, expected hash or Mongoid::Criteria, got #{data.class}"
+          end
+
+          normalized_queries[name] = details
+          normalized_queries.with_indifferent_access
+        end
+      end
+
+      # Validate that all the queries are well formed.
+      # We could do this when building them,
+      # but doing it afterward allows us to present a single, comprehensive error message
+      # (in case multiple queries have problems).
+      #
+      # @param queries a set of normalized queries
+      #
+      # @raises ArgumentError if any of the queries are missing Ruby class or database collection info
+      #
+      # @returns true if the queries are well-formatted
+      def validate_queries!(queries)
+        errors = {:collection => [], :klass => []}
+        queries.each_pair do |name, details|
+          errors[:collection] << name unless details[:collection]
+          errors[:klass] << name unless details[:klass]
+        end
+        error_text = ""
+        error_text += "Missing collection details: #{errors[:collection].join(", ")}." if errors[:collection].length > 0
+        error_text += "Missing Ruby class details: #{errors[:klass].join(", ")}." if errors[:klass].length > 0
+        if error_text.length > 0
+          raise ArgumentError, "Unable to execute multiquery. #{error_text}"
+        end
+        true
+      end
+
+      # Prepare normalized queries for use in Mongo.
+      # Currently, this involves deleting parameters that can't be
+      # turned into BSON.
+      # (We can't act directly on the normalized queries,
+      # since they contain data used later to rehydrate models.)
+      #
+      # @param queries previously normalized queries
+      #
+      # @returns [Hash] a set of queries that can be passed to MongoScript#execute
+      def mongoize_queries(queries)
+        # delete any information not needed by/compatible with Mongoid execution
+        mongoized_queries = queries.dup
+        mongoized_queries.each_pair do |name, details|
+          # have to dup the query details to avoid changing the original hashes
+          mongoized_queries[name] = details.dup.tap {|t| t.delete(:klass) }
+        end
+      end
+
+      # If any results failed, create appropriate QueryFailedError objects.
+      #
+      # @param results the results of the multiquery Javascript routine
+      # @param queries the original queries
+      #
+      # @returns the multiquery results, with any error hashes replaced by proper Ruby objects
+      def process_results(results, queries)
+        processed_results = {}
+        results.each_pair do |name, response|
+          processed_results[name] = if response.is_a?(Hash) && response["error"]
+            QueryFailedError.new(name, queries[name], response)
+          elsif response
+            # turn all the individual responses into real objects
+            response.map {|data| MongoScript.rehydrate(queries[name][:klass], data)}
+          end
+        end
+        processed_results
+      end
+    end
+  end
+end

data/lib/mongoscript/orm/mongoid_adapter.rb

@@ -0,0 +1,82 @@
+module MongoScript
+  module ORM
+    module MongoidAdapter
+
+      def self.included(base)
+        base.class_eval do
+          extend MongoScript::ORM::MongoidAdapter::ClassMethods
+        end
+      end
+
+      module ClassMethods
+        def database
+          Mongoid::Config.database
+        end
+
+        def rehydrate(klass, data)
+          Mongoid::Factory.from_db(klass, data)
+        end
+
+        # This resolves an array of Javascript arguments into
+        # hashes that can be used with MongoScript.execute.
+        # In particular, it turns Mongoid complex criteria (_id.in => ...)
+        # into regular Mongo-style hashes.
+        #
+        # @params args an array of arguments
+        #
+        # @returns an array in which all hashes have been expanded.
+        def resolve_arguments(args)
+          args.map {|arg| arg.is_a?(Hash) ? resolve_complex_criteria(arg) : arg}
+        end
+
+        # Recursively make sure any Mongoid complex critiera (like :_id.in => ...)
+        # are expanded into regular hashes (see criteria_helpers.rb in Mongoid).
+        # The built-in function only goes one level in, which doesn't work
+        # for hashes containing multiple levels with Mongoid helpers.
+        # (Am I missing where Mongoid handles this?)
+        #
+        # @note this doesn't (yet) check for circular dependencies, so don't use them!
+        #
+        # @params hash a hash that can contain Mongo-style
+        #
+        # @returns a hash that maps directly to Mongo query parameters,
+        #          which can be used by the Mongo DB driver
+        def resolve_complex_criteria(hash)
+          result = {}
+          hash.expand_complex_criteria.each_pair do |k, v|
+            result[k] = v.is_a?(Hash) ? resolve_complex_criteria(v) : v
+          end
+          result
+        end
+
+        # Turn a Mongoid::Criteria into a hash useful for multiquery.
+        #
+        # @param criteria any Mongoid::Criteria object
+        #
+        # @returns a hash containing the extracted information ready for use in multiquery
+        def build_multiquery_parameters(criteria)
+          if criteria.is_a?(Mongoid::Criteria)
+            opts = criteria.options.dup
+            # make sure the sort options are in a Mongo-compatible format
+            opts[:sort] = Mongo::Support::array_as_sort_parameters(opts[:sort] || [])
+            {
+              :selector => criteria.selector,
+              :collection => criteria.collection.name,
+              # used for rehydration
+              :klass => criteria.klass,
+              # fields are specified as a second parameter to the db[collection].find JS call
+              :fields => opts[:fields],
+              # everything in the options besides fields should be a modifier
+              # i.e. a function that can be applied via a method to a db[collection].find query
+              :modifiers => opts.tap { |o| o.delete(:fields) }
+            }
+          end
+        end
+
+        def processable_into_parameters?(object)
+          object.is_a?(Mongoid::Criteria)
+        end
+      end
+    end
+  end
+end

data/lib/mongoscript/orm/mongoid_document_methods.rb

@@ -0,0 +1,11 @@
+# TBD: do we need this?
+module MongoidDocumentMethods
+  module ClassMethods
+    def find_by_javascript(script_name, *args)
+      args = args.unshift(script_name)
+      # get a bunch of results via a Mongoid Javascript call,
+      # then turn each hash result into a Mongoid document
+      MongoScript.execute_readonly_routine(*args).map { |hash| rehydrate(self, hash) }
+    end
+  end
+end

data/lib/mongoscript/version.rb

@@ -0,0 +1,3 @@
+module Mongoscript
+  VERSION = "0.0.8"
+end

data/lib/mongoscript.rb

@@ -0,0 +1,28 @@
+require "mongoscript/orm/mongoid_adapter"
+require "mongoscript/version"
+require "mongoscript/execution"
+require 'mongoscript/multiquery'
+
+module MongoScript
+  class NoORMError < StandardError; end
+
+  # Returns the MongoScript adapter module for
+  # whichever Mongo ORM is loaded (Mongoid, MongoMapper).
+  #
+  # @note: currently only Mongoid is supported.
+  #
+  # @raises NoORMError if no ORM module can be detected.
+  #
+  # @returns MongoScript::ORM::Mongoid if Mongoid is detected
+  def self.orm_adapter
+    if const_defined? "Mongoid"
+      MongoScript::ORM::MongoidAdapter
+    else
+      raise NoORMError, "Unable to locate Mongoid!"
+    end
+  end
+
+  include orm_adapter
+  include Execution
+  include Multiquery
+end

data/mongoscript.gemspec

@@ -0,0 +1,20 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/mongoscript/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Alex Koppel"]
+  gem.email         = ["alex+git@alexkoppel.com"]
+  gem.description   = %q{An experimental Ruby library for running serverside Javascript in MongoDB.}
+  gem.summary       = %q{An experimental Ruby library for running serverside Javascript in MongoDB.}
+  gem.homepage      = ""
+
+  gem.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.files         = `git ls-files`.split("\n")
+  gem.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.name          = "mongoscript"
+  gem.require_paths = ["lib"]
+  gem.version       = Mongoscript::VERSION
+
+  # we use activesupport's with_indifferent_access
+  gem.add_runtime_dependency(%q<activesupport>, ["~> 3.0"])
+end

data/readme.md

@@ -0,0 +1,22 @@
+#MongoScript#
+
+An experimental library designed to make server-side Javascript execution in MongoDB easy, fun, and profitable.
+
+###Hey kids, this toy is for novelty use only###
+
+This library isn't "experimental" only in the sense that I'm trying out new code to see where it leads -- MongoScript is a thought experiment more than a production helper, for one simple reason.  As the MongoDB [server-side code execution](http://www.mongodb.org/display/DOCS/Server-side+Code+Execution) puts it:
+
+> Note also that [Javascript] eval doesn't work with sharding. If you expect your system to later be sharded, it's probably best to avoid eval altogether.
+
+If you're building a small-to-medium-sized system that you know will never grow huge, you may be able to use MongoScript to very cool effect.  I wouldn't recommend it for any kind of "we're gonna scale it  to the moon!" kind of product, though.  Disentangling Javascript functions and reimplementing them in Ruby so you can shard your growing database doesn't sound like my kind of fun.
+
+###Performance###
+
+There's also no guarantee that using Javascript to perform semi-complex queries is actually worth it -- Javascript can be significantly slower.  I'll post some performance statistics soon.
+
+###The cool stuff###
+
+You understand all that, you just want to hack some server-side code for the fun of it?  Let's get to it!
+
+
+####More readme coming soon!####