hadoop-rubydsl 0.0.1

data/README

@@ -0,0 +1,53 @@
+= hadoop-rubydsl
+
+== Description
+HadoopのMapper/ReducerをRubyによるDSLで記述することができます。
+hadoop-ruby.jarを利用します。
+
+例）
+apachelog.rb
+
+# log:
+#   127.0.0.1 - frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326
+#   127.0.0.1 - frank2 [10/Oct/2000:13:55:36 -0700] "GET /apache_pb2.gif HTTP/1.0" 200 2326
+#   127.0.0.1 - frank2 [10/Oct/2000:13:55:36 -0700] "GET /apache_pb3.gif HTTP/1.0" 404 2326
+
+use 'LogAnalysis'
+data.pattern /(.*) (.*) (.*) (\[.*\]) (".*") (\d*) (\d*)/
+column[2].count_uniq
+column[3].count_uniq
+column[4].count_uniq
+column[5].count_uniq
+column[6].sum
+
+=> 
+col2    frank   1
+col2    frank2  2
+col3    [10/Oct/2000:13:55:36 -0700]    3
+col4    "GET /apache_pb.gif HTTP/1.0"   1
+col4    "GET /apache_pb2.gif HTTP/1.0"  1
+col4    "GET /apache_pb3.gif HTTP/1.0"  1
+col5    200     2
+col5    404     1
+col6    6978
+
+== Usage
+0. HADOOP_HOMEを正しく設定し、Hadoopを一式立ち上げておく。
+
+1. jruby-complete-*.jar を lib/java 以下にコピー
+ex)
+$ wget http://jruby.kenai.com/downloads/1.4.0RC2/jruby-complete-1.4.0RC2.jar
+$ cp jruby-complete-*.jar lib/java/
+
+2. データを HDFS にアップロード
+ex)
+$ hadoop dfs -copyFromLocal apachelog inputs/
+
+3. MapReduce実行
+$ bin/hadoop-ruby.sh examples/apachelog.rb inputs outputs
+
+== Author
+Koichi Fujikawa <fujibee@gmail.com>
+
+== Copyright
+License: Apache License

data/Rakefile

@@ -0,0 +1,18 @@
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gemspec|
+    gemspec.name = "hadoop-rubydsl"
+    gemspec.summary = "Hadoop Ruby DSL"
+    gemspec.description = "Hadoop Ruby DSL"
+    gemspec.email = "fujibee@gmail.com"
+    gemspec.homepage = "http://github.com/fujibee/hadoop-rubydsl"
+    gemspec.authors = ["Koichi Fujikawa"]
+
+    gemspec.add_dependency 'jruby-on-hadoop'
+    gemspec.files.exclude "spec/**/*"
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler not available. Install it with: gem install jeweler"
+end
+

data/TODO

@@ -0,0 +1,2 @@
+* entire error handling
+* "use" method not allowed double quote..

data/VERSION

@@ -0,0 +1 @@
+0.0.1

data/bin/hadoop

@@ -0,0 +1,276 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# The Hadoop command script
+#
+# Environment Variables
+#
+#   JAVA_HOME        The java implementation to use.  Overrides JAVA_HOME.
+#
+#   HADOOP_CLASSPATH Extra Java CLASSPATH entries.
+#
+#   HADOOP_HEAPSIZE  The maximum amount of heap to use, in MB. 
+#                    Default is 1000.
+#
+#   HADOOP_OPTS      Extra Java runtime options.
+#   
+#   HADOOP_NAMENODE_OPTS       These options are added to HADOOP_OPTS 
+#   HADOOP_CLIENT_OPTS         when the respective command is run.
+#   HADOOP_{COMMAND}_OPTS etc  HADOOP_JT_OPTS applies to JobTracker 
+#                              for e.g.  HADOOP_CLIENT_OPTS applies to 
+#                              more than one command (fs, dfs, fsck, 
+#                              dfsadmin etc)  
+#
+#   HADOOP_CONF_DIR  Alternate conf dir. Default is ${HADOOP_HOME}/conf.
+#
+#   HADOOP_ROOT_LOGGER The root appender. Default is INFO,console
+#
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+if [ -f "$bin"/hadoop-config.sh ]; then
+  . "$bin"/hadoop-config.sh
+fi
+
+cygwin=false
+case "`uname`" in
+CYGWIN*) cygwin=true;;
+esac
+
+# if no args specified, show usage
+if [ $# = 0 ]; then
+  echo "Usage: hadoop [--config confdir] COMMAND"
+  echo "where COMMAND is one of:"
+  echo "  namenode -format     format the DFS filesystem"
+  echo "  secondarynamenode    run the DFS secondary namenode"
+  echo "  namenode             run the DFS namenode"
+  echo "  datanode             run a DFS datanode"
+  echo "  dfsadmin             run a DFS admin client"
+  echo "  fsck                 run a DFS filesystem checking utility"
+  echo "  fs                   run a generic filesystem user client"
+  echo "  balancer             run a cluster balancing utility"
+  echo "  jobtracker           run the MapReduce job Tracker node" 
+  echo "  pipes                run a Pipes job"
+  echo "  tasktracker          run a MapReduce task Tracker node" 
+  echo "  job                  manipulate MapReduce jobs"
+  echo "  queue                get information regarding JobQueues" 
+  echo "  version              print the version"
+  echo "  jar <jar>            run a jar file"
+  echo "  distcp <srcurl> <desturl> copy file or directories recursively"
+  echo "  archive -archiveName NAME <src>* <dest> create a hadoop archive"
+  echo "  daemonlog            get/set the log level for each daemon"
+  echo " or"
+  echo "  CLASSNAME            run the class named CLASSNAME"
+  echo "Most commands print help when invoked w/o parameters."
+  exit 1
+fi
+
+# get arguments
+COMMAND=$1
+shift
+
+if [ -f "${HADOOP_CONF_DIR}/hadoop-env.sh" ]; then
+  . "${HADOOP_CONF_DIR}/hadoop-env.sh"
+fi
+
+# some Java parameters
+if [ "$JAVA_HOME" != "" ]; then
+  #echo "run java in $JAVA_HOME"
+  JAVA_HOME=$JAVA_HOME
+fi
+  
+if [ "$JAVA_HOME" = "" ]; then
+  echo "Error: JAVA_HOME is not set."
+  exit 1
+fi
+
+JAVA=$JAVA_HOME/bin/java
+JAVA_HEAP_MAX=-Xmx1000m 
+
+# check envvars which might override default args
+if [ "$HADOOP_HEAPSIZE" != "" ]; then
+  #echo "run with heapsize $HADOOP_HEAPSIZE"
+  JAVA_HEAP_MAX="-Xmx""$HADOOP_HEAPSIZE""m"
+  #echo $JAVA_HEAP_MAX
+fi
+
+# CLASSPATH initially contains $HADOOP_CONF_DIR
+CLASSPATH="${HADOOP_CONF_DIR}"
+CLASSPATH=${CLASSPATH}:$JAVA_HOME/lib/tools.jar
+
+# for developers, add Hadoop classes to CLASSPATH
+if [ -d "$HADOOP_HOME/build/classes" ]; then
+  CLASSPATH=${CLASSPATH}:$HADOOP_HOME/build/classes
+fi
+if [ -d "$HADOOP_HOME/build/webapps" ]; then
+  CLASSPATH=${CLASSPATH}:$HADOOP_HOME/build
+fi
+if [ -d "$HADOOP_HOME/build/test/classes" ]; then
+  CLASSPATH=${CLASSPATH}:$HADOOP_HOME/build/test/classes
+fi
+if [ -d "$HADOOP_HOME/build/tools" ]; then
+  CLASSPATH=${CLASSPATH}:$HADOOP_HOME/build/tools
+fi
+
+# so that filenames w/ spaces are handled correctly in loops below
+IFS=
+
+# for releases, add core hadoop jar & webapps to CLASSPATH
+if [ -d "$HADOOP_HOME/webapps" ]; then
+  CLASSPATH=${CLASSPATH}:$HADOOP_HOME
+fi
+for f in $HADOOP_HOME/hadoop-*-core.jar; do
+  CLASSPATH=${CLASSPATH}:$f;
+done
+
+# add libs to CLASSPATH
+for f in $HADOOP_HOME/lib/*.jar; do
+  CLASSPATH=${CLASSPATH}:$f;
+done
+
+for f in $HADOOP_HOME/lib/jetty-ext/*.jar; do
+  CLASSPATH=${CLASSPATH}:$f;
+done
+
+for f in $HADOOP_HOME/hadoop-*-tools.jar; do
+  TOOL_PATH=${TOOL_PATH}:$f;
+done
+for f in $HADOOP_HOME/build/hadoop-*-tools.jar; do
+  TOOL_PATH=${TOOL_PATH}:$f;
+done
+
+# add user-specified CLASSPATH last
+if [ "$HADOOP_CLASSPATH" != "" ]; then
+  CLASSPATH=${CLASSPATH}:${HADOOP_CLASSPATH}
+fi
+
+# default log directory & file
+if [ "$HADOOP_LOG_DIR" = "" ]; then
+  HADOOP_LOG_DIR="$HADOOP_HOME/logs"
+fi
+if [ "$HADOOP_LOGFILE" = "" ]; then
+  HADOOP_LOGFILE='hadoop.log'
+fi
+
+# restore ordinary behaviour
+unset IFS
+
+# figure out which class to run
+if [ "$COMMAND" = "namenode" ] ; then
+  CLASS='org.apache.hadoop.hdfs.server.namenode.NameNode'
+  HADOOP_OPTS="$HADOOP_OPTS $HADOOP_NAMENODE_OPTS"
+elif [ "$COMMAND" = "secondarynamenode" ] ; then
+  CLASS='org.apache.hadoop.hdfs.server.namenode.SecondaryNameNode'
+  HADOOP_OPTS="$HADOOP_OPTS $HADOOP_SECONDARYNAMENODE_OPTS"
+elif [ "$COMMAND" = "datanode" ] ; then
+  CLASS='org.apache.hadoop.hdfs.server.datanode.DataNode'
+  HADOOP_OPTS="$HADOOP_OPTS $HADOOP_DATANODE_OPTS"
+elif [ "$COMMAND" = "fs" ] ; then
+  CLASS=org.apache.hadoop.fs.FsShell
+  HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
+elif [ "$COMMAND" = "dfs" ] ; then
+  CLASS=org.apache.hadoop.fs.FsShell
+  HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
+elif [ "$COMMAND" = "dfsadmin" ] ; then
+  CLASS=org.apache.hadoop.hdfs.tools.DFSAdmin
+  HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
+elif [ "$COMMAND" = "fsck" ] ; then
+  CLASS=org.apache.hadoop.hdfs.tools.DFSck
+  HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
+elif [ "$COMMAND" = "balancer" ] ; then
+  CLASS=org.apache.hadoop.hdfs.server.balancer.Balancer
+  HADOOP_OPTS="$HADOOP_OPTS $HADOOP_BALANCER_OPTS"
+elif [ "$COMMAND" = "jobtracker" ] ; then
+  CLASS=org.apache.hadoop.mapred.JobTracker
+  HADOOP_OPTS="$HADOOP_OPTS $HADOOP_JOBTRACKER_OPTS"
+elif [ "$COMMAND" = "tasktracker" ] ; then
+  CLASS=org.apache.hadoop.mapred.TaskTracker
+  HADOOP_OPTS="$HADOOP_OPTS $HADOOP_TASKTRACKER_OPTS"
+elif [ "$COMMAND" = "job" ] ; then
+  CLASS=org.apache.hadoop.mapred.JobClient
+elif [ "$COMMAND" = "queue" ] ; then
+  CLASS=org.apache.hadoop.mapred.JobQueueClient
+elif [ "$COMMAND" = "pipes" ] ; then
+  CLASS=org.apache.hadoop.mapred.pipes.Submitter
+  HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
+elif [ "$COMMAND" = "version" ] ; then
+  CLASS=org.apache.hadoop.util.VersionInfo
+  HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
+elif [ "$COMMAND" = "jar" ] ; then
+  CLASS=org.apache.hadoop.mapred.JobShell
+elif [ "$COMMAND" = "distcp" ] ; then
+  CLASS=org.apache.hadoop.tools.DistCp
+  CLASSPATH=${CLASSPATH}:${TOOL_PATH}
+  HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
+elif [ "$COMMAND" = "daemonlog" ] ; then
+  CLASS=org.apache.hadoop.log.LogLevel
+  HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
+elif [ "$COMMAND" = "archive" ] ; then
+  CLASS=org.apache.hadoop.tools.HadoopArchives
+  CLASSPATH=${CLASSPATH}:${TOOL_PATH}
+  HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
+elif [ "$COMMAND" = "sampler" ] ; then
+  CLASS=org.apache.hadoop.mapred.lib.InputSampler
+  HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
+else
+  CLASS=$COMMAND
+fi
+
+# cygwin path translation
+if $cygwin; then
+  CLASSPATH=`cygpath -p -w "$CLASSPATH"`
+  HADOOP_HOME=`cygpath -d "$HADOOP_HOME"`
+  HADOOP_LOG_DIR=`cygpath -d "$HADOOP_LOG_DIR"`
+  TOOL_PATH=`cygpath -p -w "$TOOL_PATH"`
+fi
+# setup 'java.library.path' for native-hadoop code if necessary
+JAVA_LIBRARY_PATH=''
+if [ -d "${HADOOP_HOME}/build/native" -o -d "${HADOOP_HOME}/lib/native" ]; then
+  JAVA_PLATFORM=`CLASSPATH=${CLASSPATH} ${JAVA} org.apache.hadoop.util.PlatformName | sed -e "s/ /_/g"`
+  
+  if [ -d "$HADOOP_HOME/build/native" ]; then
+    JAVA_LIBRARY_PATH=${HADOOP_HOME}/build/native/${JAVA_PLATFORM}/lib
+  fi
+  
+  if [ -d "${HADOOP_HOME}/lib/native" ]; then
+    if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then
+      JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:${HADOOP_HOME}/lib/native/${JAVA_PLATFORM}
+    else
+      JAVA_LIBRARY_PATH=${HADOOP_HOME}/lib/native/${JAVA_PLATFORM}
+    fi
+  fi
+fi
+
+# cygwin path translation
+if $cygwin; then
+  JAVA_LIBRARY_PATH=`cygpath -p "$JAVA_LIBRARY_PATH"`
+fi
+
+HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.dir=$HADOOP_LOG_DIR"
+HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.file=$HADOOP_LOGFILE"
+HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.home.dir=$HADOOP_HOME"
+HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.id.str=$HADOOP_IDENT_STRING"
+HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.root.logger=${HADOOP_ROOT_LOGGER:-INFO,console}"
+if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then
+  HADOOP_OPTS="$HADOOP_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH"
+fi  
+
+# run it
+#echo exec "$JAVA" $JAVA_HEAP_MAX $HADOOP_OPTS -classpath "$CLASSPATH" $CLASS "$@"
+exec "$JAVA" $JAVA_HEAP_MAX $HADOOP_OPTS -classpath "$CLASSPATH" $CLASS "$@"

data/bin/hadoop-ruby.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+BIN_DIR=`dirname "$0"`
+BASE_DIR=`cd $BIN_DIR/..; pwd`
+
+# choose hadoop sh
+HADOOP=$HADOOP_HOME/bin/hadoop 
+if [ ! -f $HADOOP ]; then
+  HADOOP=$BIN_DIR/hadoop
+  #HADOOP_OPTS="--config $BASE_DIR/conf"
+fi
+
+# fetch jruby jar if not exist
+LIB_DIR=$BASE_DIR/lib/java
+JRUBY_JAR=jruby-complete-1.4.0.jar
+if [ ! -f "$LIB_DIR/$JRUBY_JAR" ]; then
+  wget http://jruby.kenai.com/downloads/1.4.0/jruby-complete-1.4.0.jar 
+  mv $JRUBY_JAR $LIB_DIR/
+fi
+
+# construct command line
+HADOOP_RUBY_LIB_DIR=$BASE_DIR/lib
+export HADOOP_CLASSPATH=$HADOOP_RUBY_LIB_DIR
+for x in `ls $HADOOP_RUBY_LIB_DIR`; do
+  DSL_FILES=$HADOOP_RUBY_LIB_DIR/$x,$DSL_FILES
+done
+DSL_FILES=$DSL_FILES$1
+
+# execute hadoop ruby
+echo runnig $1...
+$HADOOP $HADOOP_OPTS jar $LIB_DIR/hadoop-ruby.jar org.apache.hadoop.ruby.JRubyJobRunner -libjars $LIB_DIR/$JRUBY_JAR -files $DSL_FILES $1 $2 $3

data/conf/hadoop-site.xml

@@ -0,0 +1,19 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+    <property>
+        <name>fs.default.name</name>
+        <value>hdfs://localhost:9000/</value>
+    </property>
+    <property>
+        <name>mapred.job.tracker</name>
+        <value>localhost:50040</value>
+    </property>
+    <property>
+        <name>mapred.child.java.opts</name>
+        <value>-Xmx512m</value>
+    </property>
+</configuration>

data/examples/apachelog-v2-2.rb

@@ -0,0 +1,18 @@
+use 'LogAnalysis'
+
+data 'apache log on test2' do
+  from 'apachelog/inputs'
+  to 'apachelog/outputs'
+
+  # 119.63.199.8 - - [15/Nov/2009:01:18:16 +0900] "GET /ranking/game?page=31 HTTP/1.1" 200 10077 "-" "Baiduspider+(+http://www.baidu.jp/spider/)"
+  # 203.83.243.81 - - [15/Nov/2009:01:18:33 +0900] "GET /dns_zones.txt HTTP/1.1" 404 294 "-" "libwww-perl/5.65"
+
+  each_line do
+    pattern /(.*) (.*) (.*) \[(.*)\] (".*") (\d*) (\d*) (.*) "(.*)"/
+    column_name 'remote_host', 'pass', 'user', 'access_date', 'request', 'status', 'bytes', 'pass', 'ua'
+
+    topic 'ua counts', :label => 'ua' do
+      count_uniq column[:ua]
+    end
+  end
+end

data/examples/apachelog-v2.rb

@@ -0,0 +1,25 @@
+use 'LogAnalysis'
+
+data 'apache log on test1' do
+  from 'apachlog/inputs'
+  to 'apachlog/outputs'
+
+  each_line do
+    pattern /(.*) (.*) (.*) \[(.*)\] (".*") (\d*) (\d*)/
+    column_name 'remote_host', 'pass', 'user', 'access_date', 'request', 'status', 'bytes' # 各カラムにラベルをつける
+
+    topic 'which users?', :label => 'user' do
+      count_uniq column[:user]
+    end
+
+#    topic 'access date by monthly' do
+#      select_date column[:access_date], BY_MONTHLY
+#      count column[:access_date]
+#    end
+#
+#    topic 'total bytes' do
+#      select_date column[:access_date], BY_MONTHLY
+#      sum column[:bytes].to_kilobytes # / 1024
+#    end
+  end
+end

data/examples/apachelog.rb

@@ -0,0 +1,15 @@
+# Apache log analysis
+#
+# example target data:
+#   127.0.0.1 - frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326
+#   127.0.0.1 - frank2 [10/Oct/2000:13:55:36 -0700] "GET /apache_pb2.gif HTTP/1.0" 200 2326
+#   127.0.0.1 - frank2 [10/Oct/2000:13:55:36 -0700] "GET /apache_pb3.gif HTTP/1.0" 404 2326
+
+use 'LogAnalysis'
+
+data.pattern /(.*) (.*) (.*) (\[.*\]) (".*") (\d*) (\d*)/
+column[2].count_uniq
+column[3].count_uniq
+column[4].count_uniq
+column[5].count_uniq
+column[6].sum

data/examples/hive_like_test.rb

@@ -0,0 +1,14 @@
+use 'HiveLike'
+
+# hive-like/items.txt
+# apple, 3, 100
+# banana, 1, 50
+
+create_table items(item STRING, quantity INT, price INT);
+load_data "hive-like/items.txt" items;
+
+select quantity, price, item from items;
+
+# expect
+# 0  apple 3 300
+# 1  banana 1 50

data/examples/word_count_test.rb

@@ -0,0 +1,7 @@
+use 'WordCount'
+
+from 'wc/inputs'
+to 'wc/outputs'
+
+count_uniq
+total :bytes, :words, :lines

data/hadoop-rubydsl.gemspec

@@ -0,0 +1,79 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{hadoop-rubydsl}
+  s.version = "0.0.1"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Koichi Fujikawa"]
+  s.date = %q{2009-12-26}
+  s.description = %q{Hadoop Ruby DSL}
+  s.email = %q{fujibee@gmail.com}
+  s.executables = ["hadoop", "hadoop-ruby.sh"]
+  s.extra_rdoc_files = [
+    "README",
+     "TODO"
+  ]
+  s.files = [
+    "README",
+     "Rakefile",
+     "TODO",
+     "VERSION",
+     "bin/hadoop",
+     "bin/hadoop-ruby.sh",
+     "conf/hadoop-site.xml",
+     "examples/apachelog-v2-2.rb",
+     "examples/apachelog-v2.rb",
+     "examples/apachelog.rb",
+     "examples/hive_like_test.rb",
+     "examples/word_count_test.rb",
+     "hadoop-rubydsl.gemspec",
+     "lib/core.rb",
+     "lib/hive_like.rb",
+     "lib/init.rb",
+     "lib/java/.gitignore",
+     "lib/java/hadoop-ruby.jar",
+     "lib/log_analysis.rb",
+     "lib/mapred_factory.rb",
+     "lib/util.rb",
+     "lib/word_count.rb"
+  ]
+  s.homepage = %q{http://github.com/fujibee/hadoop-rubydsl}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.5}
+  s.summary = %q{Hadoop Ruby DSL}
+  s.test_files = [
+    "spec/spec_helper.rb",
+     "spec/core_spec.rb",
+     "spec/util_spec.rb",
+     "spec/mapred_factory_spec.rb",
+     "spec/word_count_spec.rb",
+     "spec/hive_like_spec.rb",
+     "spec/log_analysis_spec.rb",
+     "spec/example_spec.rb",
+     "spec/init_spec.rb",
+     "examples/apachelog-v2.rb",
+     "examples/hive_like_test.rb",
+     "examples/word_count_test.rb",
+     "examples/apachelog-v2-2.rb",
+     "examples/apachelog.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<jruby-on-hadoop>, [">= 0"])
+    else
+      s.add_dependency(%q<jruby-on-hadoop>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<jruby-on-hadoop>, [">= 0"])
+  end
+end
+

data/lib/core.rb

@@ -0,0 +1,108 @@
+require 'util'
+require 'forwardable'
+
+module HadoopDsl
+  # controller
+  class BaseMapRed
+    extend Forwardable
+
+    attr_reader :emitted
+
+    def initialize(script, model)
+      @script, @model = script, model
+      @model.controller = self
+      @emitted = []
+    end
+
+    def run
+      body = pre_process(read_file(@script))
+      eval(body, binding, @script)
+    end
+
+    def pre_process(body)
+      body # do nothing
+    end
+
+    def emit(hash) @emitted << hash end
+
+    # all DSL statements without def is processed here
+    def method_missing(method_name, *args) self end
+  end
+
+  class BaseSetup
+    def initialize(script, conf)
+      @script, @conf = script, conf
+      output_format
+    end
+
+    def run
+      body = pre_process(read_file(@script))
+      eval(body, binding, @script)
+    end
+
+    def pre_process(body)
+      body # do nothing
+    end
+
+    # do nothing
+    def output_format; end
+
+    def paths; [@from, @to] end
+
+    def from(path) @from = path end
+    def to(path) @to = path end
+
+    # all DSL statements without def is processed here
+    def method_missing(method_name, *args) self end
+  end
+
+  class BaseMapper < BaseMapRed
+    def initialize(script, model)
+      super(script, model)
+    end
+  end
+
+  class BaseReducer < BaseMapRed
+    def initialize(script, model)
+      super(script, model)
+    end
+  end
+
+  # model
+  class BaseModel
+    attr_accessor :controller
+
+    # all DSL statements without def is processed here
+    def method_missing(method_name, *args) self end
+  end
+
+  class BaseMapperModel < BaseModel
+    attr_reader :key, :value
+
+    def initialize(key, value)
+      @key, @value = key, value
+    end
+
+    # common functions
+    def identity
+      @controller.emit(@key => @value)
+    end
+  end
+
+  class BaseReducerModel < BaseModel
+    attr_reader :key, :values
+
+    def initialize(key, values)
+      @key, @values = key, values
+    end
+
+    # common functions
+    def aggregate
+      @controller.emit(@key => @values.inject {|ret, i| ret + i})
+    end
+
+    def identity
+      @values.each {|v| @controller.emit(@key => v)}
+    end
+  end
+end