marbu 0.1.0

data/lib/marbu/server_sinatra.rb

@@ -0,0 +1,168 @@
+# -*- encoding: utf-8 -*-
+
+require 'sinatra'
+require 'marbu'
+Marbu.port = '27017'
+Marbu.uri = '127.0.0.1'
+
+Mongoid.configure do |config|
+  config.master = Mongo::Connection.new.db('marbu')
+end
+
+configure :development do
+  require 'ruby-debug'
+  enable :logging, :dump_errors, :raise_errors,
+end
+
+enable :method_override
+
+dir = File.dirname(File.expand_path(__FILE__))
+set :views, "#{dir}/server/views"
+set :public_folder, "#{dir}/server/public"
+set :static, true
+
+helpers do
+  def url_path(*path_parts)
+    [ path_prefix, path_parts ].join("/").squeeze('/')
+  end
+  alias_method :u, :url_path
+
+  def path_prefix
+    request.env['SCRIPT_NAME']
+  end
+end
+
+get "/" do
+  @mrms = Marbu::Models::Db::MongoDb.all
+  show 'root'
+end
+
+get "/builder/new" do
+  @mrm              = Marbu::Models::Db::MongoDb.new
+  @mrf              = @mrm.map_reduce_finalize
+  show 'builder'
+end
+
+post "/builder" do
+  @mrm                  = build_mrm_from_params(params.merge({:logger => logger}))
+  @mrm.save!
+  redirect "/builder/#{@mrm.uuid}"
+end
+
+get "/builder/:uuid" do
+  @mrm        = Marbu::Models::Db::MongoDb.first(conditions: {uuid: params['uuid']})
+  @mrf        = @mrm.map_reduce_finalize
+  show 'builder'
+end
+
+put "/builder/:uuid" do
+  @mrm                  = build_mrm_from_params(params.merge({:logger => logger}))
+  @mrm.save!
+  redirect "/builder/#{@mrm.uuid}"
+end
+
+delete "/builder/:uuid" do
+  @mrm = Marbu::Models::Db::MongoDb.first(conditions: {uuid: params['uuid']})
+  @mrm.destroy if @mrm.present?
+  redirect "/"
+end
+
+get "/mapreduce/:uuid" do
+  @mrm         = Marbu::Models::Db::MongoDb.first(conditions: {uuid: params['uuid']})
+  @mrf         = @mrm.map_reduce_finalize
+  @builder     = Marbu::Builder.new(@mrf)
+  @error      = nil
+
+  begin
+    # TODO: naturally this has to take the DATABASE and COLLECTION from the mapreducefilter object
+    # TODO: don;t take the parameters from the mapreducefilter object if DATABASE or DATABASE and COLLECTION are
+    # TODo: defined in the configuration (security)
+    @res = @mrf.misc.collection.map_reduce( @builder.map(:mongodb), @builder.reduce(:mongodb),
+                                            {
+                                                :query  => @builder.query,
+                                                :out    => {:replace => "tmp."+@mrm.map_reduce_finalize.misc.output_collection},
+                                                :finalize => @builder.finalize(:mongodb)
+                                            }
+    )
+  rescue Mongo::OperationFailure => e
+    @parsed_error = Marbu::Models::Db::MongoDb::Exception.explain(e, @mrf)
+    @error        = @parsed_error[:message]
+    @fix_link     = Marbu::Models::ExceptionLink.get_exception_fix_link(@parsed_error[:id], params['uuid'])
+  end
+
+  show 'mapreduce'
+end
+
+def build_mrm_from_params(params)
+  name                      = 'name'
+  function                  = 'function'
+
+  if (uuid = params['uuid'])
+    mrm                       = Marbu::Models::Db::MongoDb.find_or_create_by(uuid: uuid)
+  else
+    mrm = Marbu::Models::Db::MongoDb.new
+  end
+  mrm.name                  = params['name']
+  mrf                       = mrm.map_reduce_finalize
+
+  mrf.map                   = Marbu::Models::Map.new(
+      :code => {:text => params['map_code']}
+  )
+  mrf.reduce                = Marbu::Models::Reduce.new(
+      :code => {:text => params['reduce_code']}
+  )
+  mrf.finalize              = Marbu::Models::Finalize.new(
+      :code => {:text => params['finalize_code']}
+  )
+  mrf.query                 = Marbu::Models::Query.new(
+      :condition    => params['query_condition'],
+      :force_query  => params['query_force_query']
+  )
+  mrf.misc                  = Marbu::Models::Misc.new(
+      :database           => params['database'],
+      :input_collection   => params['input_collection'],
+      :output_collection  => params['output_collection']
+  )
+
+  # add params to map_new, reduce_new, finalize_new
+  ['map', 'reduce', 'finalize'].each do |stage|
+    ['key', 'value'].each do |type|
+      stage_type_name       = params["#{stage}_#{type}_#{name}"]
+      stage_type_function   = params["#{stage}_#{type}_#{function}"]
+
+      if( stage_type_name.present?)
+        stage_type_name.each_with_index do |n, i|
+          case stage
+            when 'map'      then add(mrf.map, type, n, stage_type_function[i])
+            when 'reduce'   then add(mrf.reduce, type, n, stage_type_function[i])
+            when 'finalize' then add(mrf.finalize, type, n, stage_type_function[i])
+            else raise Exception.new("#{stage} in #{k} is unknown")
+          end
+        end
+      end
+    end
+  end
+
+  # TODO:
+  # copy keys and values from map to reduce block
+  # copy keys from  map to finalize block
+  # reduce_new.keys       = mrf.map.keys
+  # reduce_new.values     = mrf.map.values
+  # finalize_new.keys     = mrf.map.keys
+
+  mrm.map_reduce_finalize = mrf
+  return mrm
+end
+
+def add(model, type, name, function)
+  case type
+    when 'key'      then model.add_key(name, function)
+    when 'value'    then model.add_value(name, function)
+    else raise Exception.new("#{type} is unknown")
+  end
+end
+
+def show(page)
+  haml page.to_sym
+end
+

data/lib/marbu/version.rb

@@ -0,0 +1,3 @@
+module Marbu
+  VERSION = "0.1.0"
+end

data/lib/marbu.rb

@@ -0,0 +1,65 @@
+require 'mongo'
+require 'mongoid'
+require 'core_ext/enumerable'
+
+require 'marbu/exceptions'
+require 'marbu/formatters/formatters'
+require 'marbu/builders/mongodb'
+require 'marbu/builder'
+require 'marbu/models/models'
+
+module Marbu
+  # Your code goes here...
+  extend self
+
+  attr_reader :port, :uri
+  attr_accessor :storage
+
+  def database=(database)
+    @database = database
+  end
+
+  def collection=(collection)
+    @collection = collection
+  end
+
+  def port=(port)
+    @port = port
+  end
+
+  def uri=(uri)
+    @uri = uri
+  end
+
+  def connection
+    if( @uri and @port )
+      @connection ||= Mongo::Connection.new(@uri, @port)
+    else
+      @connection = nil
+    end
+
+    @connection
+  end
+
+  def database
+    if connection and @database
+      connection.db(@database)
+    else
+      nil
+    end
+  end
+
+  def collection
+    if database and @collection
+      database.collection(@collection)
+    else
+      nil
+    end
+  end
+
+  def storage_collection
+    Mongo::Connection.new(storage[:uri],storage[:port]).db(storage[:database]).collection(storage[:collection])
+  end
+end
+
+

data/spec/constants.rb

@@ -0,0 +1,149 @@
+MR_WWB_LOC_DIM0 = {
+  :map => {
+    :keys => [
+      {:name => "DIM_LOC_0",              :function => "value.DIM_LOC_0"}
+    ],
+    :values =>  [
+      {:name => "DIM_LOC_0",              :function => "value.DIM_LOC_0"},
+      {:name => "count"}
+    ],
+    :code => {
+      :text => 'count = 1;',
+      :type => "JS"
+    }
+  },
+
+  :reduce => {
+    :values =>  [
+      {:name => "DIM_LOC_0",              :function => "value.DIM_LOC_0"},
+      {:name => "count"}
+    ],
+    :code => {
+      :type => "JS",
+      :text =>  <<-JS
+                  var count = 0;
+                  values.forEach(function(v){
+                    count += v.count;
+                  });
+                JS
+    }
+  },
+
+  :finalize => {
+    :values => [
+        {:name => "DIM_LOC_0",            :function => "value.DIM_LOC_0"},
+        {:name => "count",                :function => "value.count"}
+    ],
+    :code => {
+      :text => 'count = (count/100)*100.0;',
+      :type => "JS"
+    }
+  },
+
+  :misc => {
+    :database             => "qed_test",
+    :input_collection     => 'world_wide_business',
+    :output_collection    => 'world_wide_business_mr_dim0',
+    :output_operation     => 'replace',
+    :filter_data          => true,
+  },
+
+  :query => {
+    :datetime_fields => [:created_at]
+  }
+}
+
+MR_WWB_LOC_DIM0_NO_MAP_EMIT_KEYS = {
+  :map => {
+    :values =>  [
+      {:name => "DIM_LOC_0",            :function => "value.DIM_LOC_0"},
+      {:name => "count"}
+    ],
+    :code => {
+      :text => 'count = 1;',
+      :type => "JSS"
+    }
+  },
+
+  :reduce => {
+    :code => {
+      :text =>  <<-JS
+                  var count = 0;
+                  values.forEach(function(v){
+                    count += v.count;
+                  });
+                JS
+    }
+  },
+
+  :finalize => {
+    :values => [
+      {:name => "DIM_LOC_0",            :function => "value.DIM_LOC_0"},
+      {:name => "count",                :function => "value.count"}
+    ],
+    :code => {
+      :type => "JS",
+      :text => 'count = (count/100)*100.0;',
+    }
+  },
+
+  :misc => {
+    :database             => "qed_test",
+    :input_collection     => 'world_wide_business',
+    :output_collection    => 'world_wide_business_mr_dim0',
+  },
+
+  :query => {
+    :datetime_fields => [:created_at]
+  }
+}
+
+MR_WWB_LOC_DIM0_TWO_MAP_EMIT_KEYS = {
+  :map => {
+    :keys => [
+      {:name => "DIM_LOC_0",            :function => "value.DIM_LOC_0"},
+      {:name => "DIM_LOC_1",            :function => "value.DIM_LOC_1"}
+    ],
+    :values =>  [
+      {:name => "DIM_LOC_0",            :function => "value.DIM_LOC_0"},
+      {:name => "count"}
+    ],
+    :code => {
+      :text => 'count = 1;',
+      :type => "JS"
+    }
+  },
+
+  :reduce => {
+    :code => {
+      :type => "JS",
+      :text =>  <<-JS
+                  var count = 0;
+                  values.forEach(function(v){
+                    count += v.count;
+                  });
+                JS
+    }
+  },
+
+  :finalize => {
+    :values => [
+      {:name => "DIM_LOC_0",            :function => "value.DIM_LOC_0"},
+      {:name => "count",                :function => "value.count"}
+    ],
+    :code => {
+      :text => 'count = (count/100)*100.0;',
+      :type => "JS"
+    }
+  },
+
+  :misc => {
+    :database             => "qed_test",
+    :input_collection     => 'world_wide_business',
+    :output_collection    => 'world_wide_business_mr_dim0',
+  },
+
+  :query => {
+    :datetime_fields => [:created_at]
+  }
+}

data/spec/spec_helper.rb

@@ -0,0 +1,63 @@
+require 'simplecov'
+SimpleCov.start do
+  add_filter "/spec/"
+end
+
+require 'rubygems'
+require 'spork'
+
+Spork.prefork do
+  # Loading more in this block will cause your tests to run faster. However,
+  # if you change any configuration or code from libraries loaded here, you'll
+  # need to restart spork for it take effect.
+
+end
+
+Spork.each_run do
+  # This code will be run each time you run your specs.
+
+end
+
+# --- Instructions ---
+# Sort the contents of this file into a Spork.prefork and a Spork.each_run
+# block.
+#
+# The Spork.prefork block is run only once when the spork server is started.
+# You typically want to place most of your (slow) initializer code in here, in
+# particular, require'ing any 3rd-party gems that you don't normally modify
+# during development.
+#
+# The Spork.each_run block is run each time you run your specs.  In case you
+# need to load files that tend to change during development, require them here.
+# With Rails, your application modules are loaded automatically, so sometimes
+# this block can remain empty.
+#
+# Note: You can modify files loaded *from* the Spork.each_run block without
+# restarting the spork server.  However, this file itself will not be reloaded,
+# so if you change any of the code inside the each_run block, you still need to
+# restart the server.  In general, if you have non-trivial code in this file,
+# it's advisable to move it into a separate file so you can easily edit it
+# without restarting spork.  (For example, with RSpec, you could move
+# non-trivial code into a file spec/support/my_helper.rb, making sure that the
+# spec/support/* files are require'd from inside the each_run block.)
+#
+# Any code that is left outside the two blocks will be run during preforking
+# *and* during each_run -- that's probably not what you want.
+#
+# These instructions should self-destruct in 10 seconds.  If they don't, feel
+# free to delete them.
+
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+
+require 'marbu'
+require 'rspec'
+require 'factory_girl'
+require 'constants'
+require 'mongodb-testdata-factory'
+FactoryGirl.find_definitions
+
+RSpec.configure do |config|
+  config.include Factory::Syntax::Methods
+end

data/spec/unit/builders/builder_spec.rb

@@ -0,0 +1,5 @@
+require 'spec_helper'
+
+describe Marbu::Builder do
+
+end

data/spec/unit/builders/mongodb_spec.rb

@@ -0,0 +1,52 @@
+require 'spec_helper'
+
+MONGODB_MAP_FUNCTION_ONE_LINE                 =
+  "function(){value=this.value;count=1;emit({DIM_LOC_0:value.DIM_LOC_0},{DIM_LOC_0:value.DIM_LOC_0,count:count});}"
+MONGODB_MAP_FUNCTION_TWO_EMIT_KEYS_ONE_LINE   =
+  "function(){value=this.value;count=1;emit({DIM_LOC_0:value.DIM_LOC_0,DIM_LOC_1:value.DIM_LOC_1},{DIM_LOC_0:value.DIM_LOC_0,count:count});}"
+MONGODB_REDUCE_FUNCTION_ONE_LINE              =
+  "function(key,values){value=values[0];varcount=0;values.forEach(function(v){count+=v.count;});return{DIM_LOC_0:value.DIM_LOC_0,count:count};}"
+MONGODB_FINALIZE_FUNCTION_ONE_LINE            =
+  "function(key,value){count=(count/100)*100.0;return{DIM_LOC_0:value.DIM_LOC_0,count:value.count};}"
+
+describe Marbu::Builder::Mongodb do
+  context "for a mapreduce with one emit key" do
+    let(:mrf) { Marbu::Models::MapReduceFinalize.new(MR_WWB_LOC_DIM0) }
+
+    it "creates a map query" do
+      Marbu::Builder::Mongodb.map(mrf.map).should_not be_empty
+    end
+
+    it 'creates the correct map query' do
+       Marbu::Formatter::OneLine.perform(Marbu::Builder::Mongodb.map(mrf.map)).should == MONGODB_MAP_FUNCTION_ONE_LINE
+    end
+
+    it "creates a reduce query" do
+      Marbu::Builder::Mongodb.reduce(mrf.reduce).should_not be_empty
+    end
+
+    it 'creates the correct reduce query' do
+      Marbu::Formatter::OneLine.perform(Marbu::Builder::Mongodb.reduce(mrf.reduce)).should == MONGODB_REDUCE_FUNCTION_ONE_LINE
+    end
+
+    it "creates a finalize query" do
+      Marbu::Builder::Mongodb.finalize(mrf.finalize).should_not be_empty
+    end
+
+    it 'create the correct finalize query' do
+      Marbu::Formatter::OneLine.perform(Marbu::Builder::Mongodb.finalize(mrf.finalize)).should == MONGODB_FINALIZE_FUNCTION_ONE_LINE
+    end
+  end
+
+  context "for a mapreduce with at least two emit keys" do
+    let(:mrf) { Marbu::Models::MapReduceFinalize.new(MR_WWB_LOC_DIM0_TWO_MAP_EMIT_KEYS) }
+
+    it "creates a map query" do
+      Marbu::Builder::Mongodb.map(mrf.map).should_not be_empty
+    end
+
+    it 'creates the correct map query' do
+      Marbu::Formatter::OneLine.perform(Marbu::Builder::Mongodb.map(mrf.map)).should == MONGODB_MAP_FUNCTION_TWO_EMIT_KEYS_ONE_LINE
+    end
+  end
+end

data/spec/unit/models/base_spec.rb

@@ -0,0 +1,33 @@
+require 'spec_helper'
+
+describe Marbu::Models::Base do
+  context "a new Base instace" do
+    let(:base) { Marbu::Models::Base.new }
+    let(:modified_key_in_function) { 'value.key' }
+
+    it "can be added a key without a function" do
+      base.add_key('some_key')
+
+      base.keys.first.name.should == 'some_key'
+      base.keys.first.function.should == 'value.some_key'
+    end
+
+    it "can be added an emit key with a function" do
+      base.add_key('some_key', 'some_function')
+
+      base.keys.size.should == 1
+      key = base.keys.first
+      key.name.should == 'some_key'
+      key.function.should == 'some_function'
+    end
+
+    # it "adds an emit_key with add_key(key)" do
+    #   # base.add_key(@key)
+    #   #
+    #   # base.keys.size.should == 1
+    #   # key = base.keys.first
+    #   # key.name.should == @key
+    #   # key.function.should == @modified_key_in_function
+    # end
+  end
+end

data/spec/unit/models/map_reduce_finalize_spec.rb

@@ -0,0 +1,28 @@
+require 'spec_helper'
+
+describe Marbu::Models::MapReduceFinalize do
+
+  context "when instantiated with no params" do
+    let(:mrf) { Marbu::Models::MapReduceFinalize.new }
+
+    it "has a map" do
+      mrf.map.should be_an_instance_of(Marbu::Models::Map)
+    end
+
+    it "has a reduce" do
+      mrf.reduce.should be_an_instance_of(Marbu::Models::Reduce)
+    end
+
+    it "has a finalize" do
+      mrf.finalize.should be_an_instance_of(Marbu::Models::Finalize)
+    end
+  end
+
+  context "when instantiated with a valid params hash" do
+    let(:mrf) { Marbu::Models::MapReduceFinalize.new(MR_WWB_LOC_DIM0) }
+
+    it "return the serializable hash" do
+      mrf.serializable_hash.should == MR_WWB_LOC_DIM0
+    end
+  end
+end

data/spec/unit/models/map_spec.rb

@@ -0,0 +1,22 @@
+require 'spec_helper'
+
+describe Marbu::Models::Map do
+  context "when instantiated with empty parameters" do
+    let(:map) do
+      Marbu::Models::Map.new
+    end
+
+    it "should have no keys" do
+      map.keys.should be_empty
+    end
+
+    it "should have no values" do
+      map.values.should be_empty
+    end
+
+    it "should have no code" do
+      map.code.present?.should be_false
+    end
+  end
+end
+

data/spec/unit/models/query_spec.rb

@@ -0,0 +1,23 @@
+require 'spec_helper'
+
+describe Marbu::Models::Query do
+  context "a new Query instace" do
+    let(:query) { Marbu::Models::Query.new }
+
+    it "returns false for present?" do
+      query.present?.should be_false
+    end
+
+    context "with at least one attribute set" do
+      it "returns true for present? with attribute condition set" do
+        query.condition = 'abc'
+        query.present?.should be_true
+      end
+
+      it "returns true for present? with attribute datetime_fields set" do
+        query.datetime_fields = ['abc']
+        query.present?.should be_true
+      end
+    end
+  end
+end