appengine-mapreduce 0.0.2 → 0.0.3

data/VERSION

@@ -1 +1 @@
-0.0.2
+0.0.3

data/lib/appengine-mapreduce.rb

@@ -1,7 +1,10 @@
 $:.unshift(File.dirname(__FILE__)) unless $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
 
+require 'appengine-mapreduce/job'
+require 'appengine-mapreduce/mappable'
+
 module AppEngine
   module MapReduce
-    VERSION = '0.0.1'
+    VERSION = '0.0.3'
   end
 end

data/lib/appengine-mapreduce/job.rb

@@ -1,12 +1,19 @@
 require 'appengine-apis/labs/taskqueue'
+require File.join(File.dirname(__FILE__), 'proc_source')
+require File.join(File.dirname(__FILE__), 'string_helpers')
 
 module AppEngine
   module MapReduce
+    JRUBY_PROXY_MAPPER_CLASS = "com.jetthoughts.appengine.tools.mapreduce.JRubyMapper"
+
     BLOBSTORE_INPUT_FORMAT = "com.google.appengine.tools.mapreduce.BlobstoreInputFormat"
     DATASTORE_INPUT_FORMAT = "com.google.appengine.tools.mapreduce.DatastoreInputFormat"
 
+    OUTPUT_ENTITY_KIND_KEY = "mapreduce.mapper.outputformat.datastoreoutputformat.entitykind"
+    MAPREDUCE_SCRIPT_KEY = "mapreduce.ruby.script"
+
     CALLBACK_KEY = "mapreduce.appengine.donecallback.url"
-    
+
     import com.google.appengine.tools.mapreduce.ConfigurationXmlUtil
     import com.google.appengine.tools.mapreduce.DatastoreInputFormat
 
@@ -16,15 +23,14 @@ module AppEngine
       attr_accessor :map, :reduce, :input_kind, :output_kind, :properties, :input_class
 
       def initialize options = {}
-        puts "Job Options: #{options.inspect}"
-
         @properties = {}
 
         %w(map reduce input_class input_kind output_kind).each do |attr_n|
-          self.send("#{attr_n}=", options.delete(attr_n)) if options[attr_n]
+          key = (attr_n.to_sym rescue attr_n) || attr_n
+          self.send("#{attr_n}=", options.delete(key)) if options[key]
         end
 
-        @properties["mapreduce.map.class"] = "com.jetthoughts.appengine.tools.mapreduce.JRubyMapper"
+        @properties["mapreduce.map.class"] = JRUBY_PROXY_MAPPER_CLASS
 
       end
 
@@ -34,16 +40,30 @@ module AppEngine
         queue.add({:method => 'POST',
                   :params => {'configuration' => conf_as_string},
                   :url => "/mapreduce/start"})
-
-        puts "Conf: #{conf_as_string}"
       end
 
       def input_kind= aKind
         @properties[DatastoreInputFormat::ENTITY_KIND_KEY] = aKind
       end
 
+      def output_kind= aKind
+        @properties[OUTPUT_ENTITY_KIND_KEY] = aKind
+      end
+
+      def map &block
+        if block
+          self.map = block
+        end
+        
+        @properties[MAPREDUCE_SCRIPT_KEY]
+      end
+
       def map= aMap
-        @properties["mapreduce.ruby.script"] = aMap
+        if aMap.is_a?(Proc)
+          aMap = serialize_block aMap
+        end
+
+        @properties[MAPREDUCE_SCRIPT_KEY] = aMap
       end
 
       def input_class= klass
@@ -51,7 +71,15 @@ module AppEngine
       end
 
       def use_blobstore_input_class
-        input_class = BLOBSTORE_INPUT_FORMAT
+        self.input_class = BLOBSTORE_INPUT_FORMAT
+      end
+
+      def self.create_job_for mapper_name
+        map = <<MAP
+require 'lib/mappers/#{mapper_name.to_s}_mapper'
+include #{mapper_name.to_s.camelize}Mapper
+MAP
+        Job.new(:map => map)
       end
 
       protected
@@ -60,6 +88,14 @@ module AppEngine
         @properties.each {|k,v| @conf.set(k, v)}
         ConfigurationXmlUtil.convertConfigurationToXml(@conf)
       end
+
+      def serialize_block block
+        <<MAP
+self.class.class_eval do
+  define_method(:map, lambda{ #{ProcSource.handle(block)} })
+end
+MAP
+      end
     end
   end
 end

data/lib/appengine-mapreduce/mappable.rb

@@ -0,0 +1,16 @@
+module AppEngine
+  module Mappable
+    def self.included(base)
+      #base.property :name, String
+      base.extend ClassMethods
+    end
+
+    module ClassMethods
+      def async_map &block
+        job = AppEngine::MapReduce::Job.new(:input_kind => name || storage_names[:default])
+        job.map = block
+        job.run
+      end
+    end
+  end
+end

data/lib/appengine-mapreduce/proc_source.rb

@@ -0,0 +1,141 @@
+require 'stringio'
+require 'irb/ruby-lex'
+
+SCRIPT_LINES__ = {} unless defined? SCRIPT_LINES__
+
+module ProcSource #:nodoc:
+  def get_lines(filename, start_line = 0)
+    return nil unless filename
+
+    case filename
+      # special "(irb)" descriptor?
+      when "(irb)"
+        IRB.conf[:MAIN_CONTEXT].io.line(start_line .. -1)
+      # special "(eval...)" descriptor?
+      when /^\(eval.+\)$/
+        EVAL_LINES__[filename][start_line .. -1]
+      # regular file
+      else
+        # Ruby already parsed this file? (see disclaimer above)
+        if lines = SCRIPT_LINES__[filename]
+          lines[(start_line - 1) .. -1]
+          # If the file exists we're going to try reading it in
+        elsif File.exist?(filename)
+          begin
+            File.readlines(filename)[(start_line - 1) .. -1]
+          rescue
+            nil
+          end
+        end
+    end
+  end
+
+  def handle(proc)
+    filename, line = source_descriptor proc
+    lines = get_lines(filename, line) || []
+
+    lexer = RubyLex.new
+    lexer.set_input(StringIO.new(lines.join))
+
+    state = :before_constructor
+    nesting_level = 1
+    start_token, end_token = nil, nil
+    found = false
+    while token = lexer.token
+      # we've not yet found any proc-constructor -- we'll try to find one.
+      if [:before_constructor, :check_more].include?(state)
+        # checking more and newline? -> done
+        if token.is_a?(RubyToken::TkNL) and state == :check_more
+          state = :done
+          break
+        end
+        # token is Proc?
+        if token.is_a?(RubyToken::TkCONSTANT) and
+                token.instance_variable_get(:@name) == "Proc"
+          # method call?
+          if lexer.token.is_a?(RubyToken::TkDOT)
+            method = lexer.token
+            # constructor?
+            if method.is_a?(RubyToken::TkIDENTIFIER) and
+                    method.instance_variable_get(:@name) == "new"
+              unless state == :check_more
+                # okay, code will follow soon.
+                state = :before_code
+              else
+                # multiple procs on one line
+                return
+              end
+            end
+          end
+          # token is lambda or proc call?
+        elsif token.is_a?(RubyToken::TkIDENTIFIER) and
+                %w{proc lambda}.include?(token.instance_variable_get(:@name))
+          unless state == :check_more
+            # okay, code will follow soon.
+            state = :before_code
+          else
+            # multiple procs on one line
+            return
+          end
+        elsif token.is_a?(RubyToken::TkfLBRACE) or token.is_a?(RubyToken::TkDO)
+          # found the code start, update state and remember current token
+          state = :in_code
+          start_token = token
+        end
+
+        # we're waiting for the code start to appear.
+      elsif state == :before_code
+        if token.is_a?(RubyToken::TkfLBRACE) or token.is_a?(RubyToken::TkDO)
+          # found the code start, update state and remember current token
+          state = :in_code
+          start_token = token
+        end
+
+        # okay, we're inside code
+      elsif state == :in_code
+        if token.is_a?(RubyToken::TkRBRACE) or token.is_a?(RubyToken::TkEND)
+          nesting_level -= 1
+          if nesting_level == 0
+            # we're done!
+            end_token = token
+            # parse another time to check if there are multiple procs on one line
+            # we can't handle that case correctly so we return no source code at all
+            state = :check_more
+          end
+        elsif token.is_a?(RubyToken::TkfLBRACE) or token.is_a?(RubyToken::TkDO) or
+                token.is_a?(RubyToken::TkBEGIN) or token.is_a?(RubyToken::TkCASE) or
+                token.is_a?(RubyToken::TkCLASS) or token.is_a?(RubyToken::TkDEF) or
+                token.is_a?(RubyToken::TkFOR) or token.is_a?(RubyToken::TkIF) or
+                token.is_a?(RubyToken::TkMODULE) or token.is_a?(RubyToken::TkUNLESS) or
+                token.is_a?(RubyToken::TkUNTIL) or token.is_a?(RubyToken::TkWHILE) or
+                token.is_a?(RubyToken::TklBEGIN)
+          nesting_level += 1
+        end
+      end
+    end
+
+    if start_token and end_token
+      start_line, end_line = start_token.line_no - 1, end_token.line_no - 1
+      source = lines[start_line .. end_line]
+      start_offset = start_token.char_no
+      start_offset += 1 if start_token.is_a?(RubyToken::TkDO)
+      end_offset = -(source.last.length - end_token.char_no)
+      source.first.slice!(0 .. start_offset)
+      source.last.slice!(end_offset .. -1)
+
+      # Can't use .strip because newline at end of code might be important
+      # (Stuff would break when somebody does proc { ... #foo\n})
+      source.join.gsub(/^ | $/, "")
+    end
+  end
+
+  def source_descriptor p
+    if md = /^#<Proc:0x[0-9A-Fa-f]+@(.+):(\d+)>$/.match(p.inspect)
+      filename, line = md.captures
+      return filename, line.to_i
+    end
+  end
+
+
+  module_function :handle, :get_lines, :source_descriptor
+end

data/lib/appengine-mapreduce/string_helpers.rb

@@ -0,0 +1,29 @@
+module StringHelper
+  extend self
+
+  def camelize(lower_case_and_underscored_word, first_letter_in_uppercase = true)
+    lower_case_and_underscored_word.to_s.gsub(/\/(.?)/) { "::#{$1.upcase}" }.gsub(/(?:^|_)(.)/) { $1.upcase }
+  end
+
+  def underscore(camel_cased_word)
+    word = camel_cased_word.to_s.dup
+    word.gsub!(/::/, '/')
+    word.gsub!(/([A-Z]+)([A-Z][a-z])/,'\1_\2')
+    word.gsub!(/([a-z\d])([A-Z])/,'\1_\2')
+    word.tr!("-", "_")
+    word.downcase!
+    word
+  end
+end
+
+class ::String
+  def camelize
+    StringHelper.camelize(self)
+  end
+end
+
+class ::Symbol
+  def camelize
+    StringHelper.camelize(self)
+  end
+end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: appengine-mapreduce
 version: !ruby/object:Gem::Version 
-  hash: 27
+  hash: 25
   prerelease: false
   segments: 
   - 0
   - 0
-  - 2
-  version: 0.0.2
+  - 3
+  version: 0.0.3
 platform: ruby
 authors: 
 - Paul Nikitochkin
@@ -49,6 +49,9 @@ files:
 - lib/appengine-mapreduce/boot.rb
 - lib/appengine-mapreduce/datastore_input_format.rb
 - lib/appengine-mapreduce/job.rb
+- lib/appengine-mapreduce/mappable.rb
+- lib/appengine-mapreduce/proc_source.rb
+- lib/appengine-mapreduce/string_helpers.rb
 - target/appengine-mapreduce-jruby-0.0.1-SNAPSHOT.jar
 - target/dependency/aopalliance-1.0.jar
 - target/dependency/appengine-mapper-0.0.1-SNAPSHOT.jar