marbu 0.1.0

data/Rakefile

@@ -0,0 +1,13 @@
+require 'bundler'
+Bundler.setup
+
+require "rake"
+require "rspec"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new("spec:unit") do |spec|
+  spec.pattern = "spec/unit/**/*_spec.rb"
+end
+
+task :spec => [ "spec:unit" ]
+task :default => :spec

data/Readme.md

@@ -0,0 +1,58 @@
+MA(p) R(educe) BU(ilder)
+========================
+
+Marbu enables you to add mapreduce code-generating functionality to your software within seconds.
+
+What does it do?
+----------------
+
+Marbu contains three destinct parts
+
+1. The MapReduceFinalize-Model which acts as an Interface for the MapReduceBuilder
+2. The map-reduce core functionality: this creates map reduce code for different map reduce systems
+3. A Sinatra App which lets you instantly play around with the builder and shows you what you can do.
+
+Installation
+-----------------
+Install with 'gem install marbu'.
+
+For bundler add "gem 'marbu'" to your Gemfile.
+
+Stand alone usage
+-----------------
+
+Marbu comes complete with a little Sinatra App to help you along. You start the app with 'marbu-web marbu-web-mongo-config.rb'.
+This will start Marbu with the provided MongoDB-Configuration file.
+
+If everything is working, you can go to 'localhost:5678' (if you are already running something on 5678 then try 5679) and
+you should see all databases in your local MongoDB installation.
+
+After selecting a Database you will see all collections within this database. You select a collection and go into the 'marbu' mode.
+In the 'marbu' mode you can build your map - reduce - finalize functionality. After that run your query and see what you get. Done!
+
+
+## Development
+
+Clone the repo
+
+    $ git clone git@github.com:QEDio/qstate.git
+
+Update submodules
+
+    $ git submodule init && git submodule update
+
+Install dependencies
+
+    $ bundle install
+
+Run the tests
+
+    $ rake
+
+Running the built-in Sinatra server can be done using the method above or using a standard config.ru file just like any other Rack-based app. Just make sure to provide the config file path through the corresponding environment variable:
+
+    $ MARBUCONFIG=bin/marbu-web-mongo-config.rb rackup
+
+You can also activate the debugger and pick a specific server if you want:
+
+    $ MARBUCONFIG=bin/marbu-web-mongo-config.rb thin --debug --rackup config.ru start

data/bin/marbu-web

@@ -0,0 +1,20 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib')
+begin
+  require 'vegas'
+rescue LoadError
+  require 'rubygems'
+  require 'vegas'
+end
+require 'marbu/server'
+
+
+Vegas::Runner.new(Marbu::Server, 'marbu-web', {
+  :before_run => lambda {|v|
+    path = (ENV['MARBUCONFIG'] || v.args.first)
+    load path.to_s.strip if path
+  }
+}) do |runner, opts, app|
+
+end

data/bin/marbu-web-mongo-config.rb

@@ -0,0 +1,8 @@
+require 'mongo'
+
+Marbu.uri             = '127.0.0.1'
+Marbu.port            = '27017'
+
+Mongoid.configure do |config|
+  config.master = Mongo::Connection.new.db('marbu')
+end

data/lib/core_ext/array.rb

@@ -0,0 +1,7 @@
+class Array
+  def uniq_by
+    hash, array = {}, []
+    each { |i| hash[yield(i)] ||= (array << i) }
+    array
+  end
+end

data/lib/core_ext/enumerable.rb

@@ -0,0 +1,25 @@
+module Enumerable
+  def mean
+    sum / size
+  end
+
+  # returns a new hash/array
+  # keeps object references in tact
+  # symbolize a hash recursivly
+  # if we encounter an array, we will also call symbolize for every hash or array found
+  def symbolize_keys_rec
+    if is_a?(Hash)
+      inject({}) do |options, (key, value)|
+        options[(key.to_sym rescue key) || key] = (value.is_a?(Hash)||value.is_a?(Array)) ? value.symbolize_keys_rec : value
+        options
+      end
+    elsif is_a?(Array)
+      inject([]) do |options, value|
+        options << ((value.is_a?(Hash)||value.is_a?(Array)) ? value.symbolize_keys_rec : value)
+        options
+      end
+    else
+      raise Exception.new("Can't do that!")
+    end
+  end
+end

data/lib/core_ext/hash.rb

@@ -0,0 +1,35 @@
+class Hash
+  def has_nested_key?(key)
+    return true if has_key?(key)
+    values.select{|v| v.is_a?(Hash)}.each do |value|
+      return value.has_nested_key?(key)
+    end
+    return false
+  end
+
+  def nested_key(key)
+    return fetch(key) if has_key?(key)
+    values.select{|v| v.is_a?(Hash)}.each do |value|
+      return value.nested_key(key)
+    end
+    return default(key)
+  end
+
+  def deep_merge(other_hash)
+    a = self.dup
+    other_hash.each do |k, v|
+      if v.is_a? Hash
+        if a.has_key? k
+          a[k] = a[k].deep_merge(v)
+        else
+          a[k] = v
+        end
+      else
+        a[k] = v
+      end
+    end
+    a
+  end
+
+end
+

data/lib/core_ext/indifferent_access.rb

@@ -0,0 +1,11 @@
+class HashWithIndifferentAccess
+  def self.deep_transform hash
+    a = HashWithIndifferentAccess.new(hash)
+    a.each do |k, v|
+      if v.is_a? Hash
+        v = deep_transform(v)
+      end
+    end
+    a
+  end
+end

data/lib/core_ext/object.rb

@@ -0,0 +1,25 @@
+module CallerMethodName
+
+  private
+
+  def current_method
+    caller[0] =~ /\d:in `([^']+)'/
+    $1
+  end
+
+  def caller_method_name
+      parse_caller(caller(2).first).last
+  end
+
+  def parse_caller(at)
+    if /^(.+?):(\d+)(?::in `(.*)')?/ =~ at
+      file = Regexp.last_match[1]
+      line = Regexp.last_match[2].to_i
+      method = Regexp.last_match[3]
+      [file, line, method]
+    end
+  end
+
+end
+
+Object.__send__(:include, CallerMethodName)

data/lib/core_ext/open_struct.rb

@@ -0,0 +1,3 @@
+# monkeypatch openstruct to get the method id working for openstructs
+# which get an id as attribute
+OpenStruct.__send__(:define_method, :id) { @table[:id] }

data/lib/core_ext/string.rb

@@ -0,0 +1,25 @@
+class String
+  # Truncates the string after the first +chars+ (100) at word boundary and
+  # adds +ending+ ("...").
+  def truncate_at_word chars=100, ending="..."
+    sub(/^(.{#{chars}}[\w.]*)(.*)/m){ $2.empty? ? $1 : $1 + ending}
+  end
+
+  # returns a url/ascii/old-systems/non-utf8 friendly representation of self
+  def callname
+    @_callname ||= dup.
+      gsub(/[Ää]+/i,'ae').
+      gsub(/[Üü]+/i,'ue').
+      gsub(/[Öö]+/i,'oe').
+      gsub(/[ß]+/i,'ss').
+      downcase.
+      gsub(/[^a-z0-9]+/i, '-').
+      gsub(/(^[-]+|[-]+$)/, '')
+  end
+
+  def to_bool
+    return true if match('true')
+    return false
+  end
+end
+

data/lib/marbu/builder.rb

@@ -0,0 +1,48 @@
+# -*- encoding: utf-8 -*-
+
+module Marbu
+  # the builder class is a lightweight container that holds a MapReduceModel and the requester Builder class
+  # its basically a front for the internals of the builders
+  class Builder
+    attr_accessor :map_reduce_model
+
+    def initialize(mrm, options = {})
+      if mrm.is_a?(Marbu::Models::MapReduceFinalize)
+        @map_reduce_model = mrm
+      elsif mrm.is_a?(Hash)
+        @map_reduce_model = Marbu::Models::MapReduceFinalize.new(mrm)
+      else
+        raise Exception.new("Parameter mrm was neither of type MapReduceFinalize nor Hash, but #{mrm.class}. Aborting")
+      end
+
+      @builder_clasz    = options[:builder] || Marbu::Builder::Mongodb
+      @formatter_clasz  = options[:formatter] || Marbu::Formatter::Dummy
+    end
+
+    def map(format = :text)
+      @formatter_clasz.perform(@builder_clasz.map(@map_reduce_model.map, format))
+    end
+
+    def reduce(format = :text)
+      @formatter_clasz.perform(@builder_clasz.reduce(@map_reduce_model.reduce, format))
+    end
+
+    def finalize(format = :text)
+      @formatter_clasz.perform(@builder_clasz.finalize(@map_reduce_model.finalize, format))
+    end
+
+    def query
+      @map_reduce_model.query.condition
+    end
+
+    def log
+      self.to_s
+    end
+
+    def to_s
+      puts "Map: #{map}"
+      puts "Reduce: #{reduce}"
+      puts "Finalize: #{finalize}"
+    end
+  end
+end

data/lib/marbu/builders/mongodb.rb

@@ -0,0 +1,106 @@
+module Marbu
+  class Builder
+    class Mongodb
+      def self.map(map, format)
+        case format
+          when :text then map_int(map)
+          when :mongodb then cleanup(map_int(map))
+        end
+      end
+
+      def self.reduce(reduce, format)
+        case format
+          when :text then self.reduce_int(reduce)
+          when :mongodb then cleanup(self.reduce_int(reduce))
+        end
+      end
+
+
+      def self.finalize(finalize, format)
+        case format
+          when :text then self.finalize_int(finalize)
+          when :mongodb then cleanup(self.finalize_int(finalize))
+        end
+      end
+
+      private
+        def self.map_int(map)
+          <<-JS
+function(){
+  #{get_value(:map)}
+  #{map.code.text}
+  #{get_emit(:map, map.values, map.keys)}
+}
+JS
+        end
+        def self.reduce_int(reduce)
+          <<-JS
+function(key,values){
+  #{get_value(:reduce)}
+  #{reduce.code.text}
+  #{get_emit(:reduce, reduce.values)}
+}
+JS
+        end
+        def self.finalize_int(finalize)
+          <<-JS
+function(key, value){
+  #{finalize.code.text}
+  #{get_emit(:finalize, finalize.values)}
+}
+JS
+        end
+
+        def self.cleanup(code)
+          code.gsub!(/[\n|\r]/,'')
+        end
+
+        def self.get_value(function)
+          case function
+            when :map        then "value=this.value;"
+            when :reduce     then "value=values[0];"
+            else raise Exception.new("Value-foo for #{function} not defined!")
+          end
+        end
+
+        def self.get_emit(function, values, keys = nil)
+          case function
+            when :map        then emit_map(keys, values)
+            when :reduce     then emit_reduce(values)
+            when :finalize   then emit_finalize(values)
+            else raise Exception.new("Emit for #{function} not defined!")
+          end
+        end
+
+        def self.emit_map(keys, values)
+          emit = "emit( #{emit_keys(keys)}, "
+          emit += emit_core(values)
+          emit += " );"
+          return emit
+        end
+
+        def self.emit_reduce(values)
+          "return " + emit_core(values) + ";"
+        end
+
+        def self.emit_finalize(values)
+          "return " + emit_core(values) + ";"
+        end
+
+        def self.emit_keys(keys)
+          emit_core(keys)
+        end
+
+        def self.emit_core(values)
+          core = "{ "
+          values.each do |v|
+            core += " #{v.name}: #{v.function||v.name},"
+          end
+          # delete last comma
+          core = core[0..core.size-2]
+          core += " }"
+          return core
+        end
+      end
+  end
+end

data/lib/marbu/exceptions.rb

@@ -0,0 +1,7 @@
+# -*- encoding: utf-8 -*-
+
+module Marbu
+  class Exceptions
+    class NoEmitKeys < RuntimeError; end
+  end
+end

data/lib/marbu/formatters/base.rb

@@ -0,0 +1,6 @@
+module Marbu
+  class Formatter
+    class Base
+    end
+  end
+end

data/lib/marbu/formatters/dummy.rb

@@ -0,0 +1,9 @@
+module Marbu
+  class Formatter
+    class Dummy < Base
+      def self.perform(str)
+        str
+      end
+    end
+  end
+end

data/lib/marbu/formatters/formatters.rb

@@ -0,0 +1,3 @@
+require 'marbu/formatters/base'
+require 'marbu/formatters/dummy'
+require 'marbu/formatters/one_line'

data/lib/marbu/formatters/one_line.rb

@@ -0,0 +1,9 @@
+module Marbu
+  class Formatter
+    class OneLine < Base
+      def self.perform(str)
+        str.gsub(/( |\n)/, '')
+      end
+    end
+  end
+end

data/lib/marbu/models/db/db.rb

@@ -0,0 +1,3 @@
+require 'marbu/models/db/mongodb/mongodb'
+require 'marbu/models/db/mongodb/exceptions'
+require 'marbu/models/db/mongodb/structure'

data/lib/marbu/models/db/mongodb/exceptions.rb

@@ -0,0 +1,59 @@
+module Marbu
+  module Models
+    module Db
+      class MongoDb
+        class Exception < StandardError
+          REFERENCE_ERROR       = 'ReferenceError'
+          NS_ERROR              = 'NS_ERROR'
+          REGEX_DB_COMMAND = { :regex => /(ns doesn't exist)/,
+                              :mapping => { :function         => {:type => :string, :value => 'Buu'},
+                                            :error_type       => {:type => :string, :value => NS_ERROR},
+                                            :variable         => {:type => :string, :value => 'Buu'},
+                                            :assertion_code   => {:type => :string, :value => 'Buu'},
+                                            :error_message    => {:type => :string, :value => 'Buu'} }}
+          #REGEX_REF_ERROR = { :regex => /(.*?)'(.*?)'(.*?): \((assertion): '(.*?) (.*?): (.*?): (ReferenceError): (.*?)(is.*?)(nofile_b)?:\d'; (assertionCode): '(.*)';.*(errmsg): '(.*?)'/,
+          #                    mapping => { :function => 5, :error_type => 8, :variable => 9, :assertion_code => 13, :error_message => 15 }}}
+          REGEXS = [ REGEX_DB_COMMAND ]
+
+          def self.explain(e, mrf)
+            parsed_exception = parse(e)
+
+            case parsed_exception[:error_type]
+              when REFERENCE_ERROR  then explain_reference_error(parsed_exception, mrf)
+              when NS_ERROR         then { :id => NS_ERROR, :message => explain_ns_error(parsed_exception, mrf)}
+
+              else raise Exception.new("The Error #{e.to_s} is unknown to me. Parsed: #{parsed_exception.inspect}")
+            end
+          end
+
+          def self.parse(e)
+            {}.tap do |parsed_exception|
+              # http://rubular.com/r/MYM9ADhz4Z
+              REGEXS.each do |regex|
+                e.result["errmsg"] =~ regex[:regex]
+                if( $1.present? )
+                  parsed_exception[:orig_error] = e
+                  [:function, :error_type, :variable, :assertion_code, :error_message].each do |key|
+                    case regex[:mapping][key][:type]
+                      when :string then parsed_exception[key]           = regex[:mapping][key][:value]
+                      else "Unknown type: #{regex[:mapping][key][:type]}"
+                    end
+                  end
+                  break
+                end
+              end
+            end
+          end
+
+          def self.explain_ns_error( parsed_exception, mrf )
+            "Either the database #{mrf.misc.database} or the collection #{mrf.misc.input_collection} is unknown."
+          end
+
+          def self.explain_reference_error( parsed_exception, mrf )
+            "The variable #{parsed_exception[:variable]} defined in function #{parsed_exception[:function]} is unknown. This either means you are trying to access a datafield that doesn't exist in the collection, or you misspelled a variable name."
+          end
+        end
+      end
+    end
+  end
+end

data/lib/marbu/models/db/mongodb/mongodb.rb

@@ -0,0 +1,54 @@
+# -*- encoding: utf-8 -*-
+
+require 'mongoid'
+require 'uuid'
+
+module Marbu
+  module Models
+    module Db
+      class MongoDb
+        include Mongoid::Document
+        store_in :marbu
+
+        field :uuid,                  type: String
+        field :name,                  type: String
+        field :map_reduce_finalize,   type: MapReduceFinalize
+
+        def initialize( ext_params = {} )
+          super
+          params                    = default_params.merge( ext_params.delete_if{|k,v| v.blank? })
+
+          self.uuid                 = params[:uuid]
+          self.name                 = params[:name]
+          self.map_reduce_finalize  = MapReduceFinalize.new( params[:map_reduce_finalize] )
+        end
+
+        def default_params
+          {
+            :name                 => "NoName00",
+            :uuid                 => UUID.new.generate(:compact),
+            :map_reduce_finalize => {
+              :map => {
+                :keys => [{:name => "map_key1"}],
+                :values => [{:name => "map_value1"}]
+              },
+              :reduce => {
+                :keys => [{:name => "reduce_key1"}],
+                :values => [{:name => "reduce_value1"}]
+              },
+              :finalize => {
+                :keys => [{:name => "finalize_key1"}],
+                :values => [{:name => "finalize"}]
+              },
+              :misc => {
+                :database           => 'database',
+                :input_collection   => 'input_collection',
+                :output_collection  => 'output_collection'
+              }
+            }
+          }
+        end
+      end
+    end
+  end
+end

data/lib/marbu/models/db/mongodb/structure.rb

@@ -0,0 +1,40 @@
+module Marbu
+  module Models
+    module Db
+      class MongoDb
+        class Structure
+          def self.get_collection_structure( collection_info )
+            raise "collection_info needs to be Marbu::Models::Misc, but is: #{collection_info.class}" unless collection_info.is_a?(Marbu::Models::Misc)
+
+            collection  = collection_info.collection
+            document    = collection.find().first()
+
+            get_collection_structure_int(document)
+          end
+
+          def self.get_first_and_last_document( collection_info )
+            raise "collection_info needs to be Marbu::Models::Misc, but is: #{collection_info.class}" unless collection_info.is_a?(Marbu::Models::Misc)
+
+            collection  = collection_info.collection
+
+            [collection.find().sort([['_id', 1]]).first(), collection.find().sort([['_id', -1]]).first()]
+          end
+
+
+          private
+            def self.get_collection_structure_int(document)
+              {}.tap do |hsh|
+                document.each_pair do |key, value|
+                  if( value.is_a?(Hash) )
+                    hsh[key] = get_collection_structure_int(value)
+                  else
+                    hsh[key] = key
+                  end
+                end
+              end
+            end
+        end
+      end
+    end
+  end
+end

data/lib/marbu/models/exception_link.rb

@@ -0,0 +1,11 @@
+module Marbu
+  module Models
+    class ExceptionLink
+      def self.get_exception_fix_link(error, uuid)
+        case error
+          when 'NS_ERROR' then "#misc"
+        end
+      end
+    end
+  end
+end

data/lib/marbu/models/models.rb

@@ -0,0 +1,3 @@
+require 'marbu/models/mrf/mrf'
+require 'marbu/models/db/db'
+require 'marbu/models/exception_link'