gigantron 0.0.1 → 0.1.0

data/Manifest.txt

@@ -15,6 +15,14 @@ app_generators/gigantron/templates/test/test_helper.rb
 bin/gigantron
 config/hoe.rb
 config/requirements.rb
+gigantron_generators/mapreduce_task/USAGE
+gigantron_generators/mapreduce_task/mapreduce_task_generator.rb
+gigantron_generators/mapreduce_task/templates/mapreduce/mr_task.rb
+gigantron_generators/mapreduce_task/templates/tasks/task.rake
+gigantron_generators/mapreduce_task/templates/test/tasks/test_task.rb
+gigantron_generators/migration/USAGE
+gigantron_generators/migration/migration_generator.rb
+gigantron_generators/migration/templates/db/migrate/migration.rb
 gigantron_generators/model/USAGE
 gigantron_generators/model/model_generator.rb
 gigantron_generators/model/templates/models/model.rb
@@ -24,6 +32,7 @@ gigantron_generators/task/task_generator.rb
 gigantron_generators/task/templates/tasks/task.rake
 gigantron_generators/task/templates/test/tasks/test_task.rb
 lib/gigantron.rb
+lib/gigantron/migrator.rb
 lib/gigantron/tasks/db.rb
 lib/gigantron/tasks/test.rb
 lib/gigantron/version.rb
@@ -39,6 +48,8 @@ test/test_generator_helper.rb
 test/test_gigantron.rb
 test/test_gigantron_generator.rb
 test/test_helper.rb
+test/test_mapreduce_task_generator.rb
+test/test_migration_generator.rb
 test/test_model_generator.rb
 test/test_task_generator.rb
 website/index.html

data/README.txt

@@ -7,7 +7,9 @@ http://github.com/schleyfox/gigantron
 
 Gigantron is a simple framework for the creation and organization of
 data processing projects. Data-processing transforms are created as Rake tasks
-and data is handled through DataMapper models.
+and data is handled through ActiveRecord* models.
+
+* Will switch back to DataMapper once it plays nice with JRuby
 
 == FEATURES/PROBLEMS:
 
@@ -36,7 +38,7 @@ to add code.
 * RubyGems
 * RubiGen
 * Rake
-* DataMapper
+* ActiveRecord
 * ActiveSupport
 * Shoulda
 
@@ -44,11 +46,16 @@ to add code.
 
 sudo gem install gigantron
 
+== HACKING:
+
+Check out the website for a quick overview of how to fiddle with the generators
+behind Gigantron. http://gigantron.rubyforge.org.
+
 == LICENSE:
 
 (The MIT License)
 
-Copyright (c) 2008 FIX
+Copyright (c) 2008 Ben Hughes
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

data/app_generators/gigantron/gigantron_generator.rb

@@ -76,5 +76,6 @@ EOS
       models
       lib
       test
+      log
     )
 end

data/app_generators/gigantron/templates/database.yml

@@ -1,9 +1,8 @@
 :real:
-  :adapter: sqlite3
-  :database: db/real.sqlite3
-  :timeout: 5000
+  :adapter: jdbcsqlite3
+  :url: jdbc:sqlite:db/real.sqlite3
 
 :test:
-  :adapter: sqlite3
-  :database: db/test.sqlite3
-  :timeout: 5000
+  :adapter: jdbcsqlite3
+  :url: jdbc:sqlite:db/test.sqlite3
+

data/app_generators/gigantron/templates/initialize.rb

@@ -15,12 +15,20 @@ require 'activesupport'
 #set up autoload paths
 Dependencies.load_paths << "#{GTRON_ROOT}/lib/"
 
-gem 'dm-core'
-require 'data_mapper'
+require 'active_record'
 
 
 def get_db_conn(env)
-  DataMapper.setup(:default, YAML::load(File.read("database.yml"))[env])
-  Dir["models/**/*.rb"].each {|r| load r }
+  env = env.to_sym
+  #set up logging
+  ActiveRecord::Base.logger = Logger.new("#{GTRON_ROOT}/log/#{env}.log")
+
+  #load in dbs from database.yml
+  ActiveRecord::Base.establish_connection(
+    YAML::load(File.read("#{GTRON_ROOT}/database.yml"))[env])
+
+  #load all models
+  Dir["#{GTRON_ROOT}/models/**/*.rb"].each {|r| load r }
+
   nil
 end

data/app_generators/gigantron/templates/tasks/import.rake

@@ -6,5 +6,5 @@ task :import do
   #
   # Ex:
   #   Foo.import_yaml(FileList["input/*.yml"].to_a)
-  get_db_conn(GTRON_ENV)
+  get_db_conn(ENV["GTRON_ENV"] || GTRON_ENV)
 end

data/app_generators/gigantron/templates/test/test_helper.rb

@@ -1,5 +1,7 @@
 require 'rubygems'
 require 'test/unit'
 require 'shoulda'
+require 'gigantron/migrator'
 require File.dirname(__FILE__) + '/../initialize'
 silence_warnings { GTRON_ENV = :test }
+ENV['GTRON_ENV'] = 'test'

data/config/hoe.rb

@@ -12,10 +12,7 @@ EXTRA_DEPENDENCIES = [
   ['rubigen', '>= 1.3.2'],
   ['rake', '>= 0.8.1'],
   ['Shoulda', '>= 1.1.1'],
-  ['dm-core', '>= 0.9.1'],
-  ['do_sqlite3', '>= 0.9.1'],
-  ['data_objects', '>= 0.9.1'],
-  ['dm-more', '>= 0.9.1']
+  ['activerecord', '>= 2.0.2'],
 ]    # An array of rubygem dependencies [name, version]
 
 @config_file = "~/.rubyforge/user-config.yml"
@@ -48,7 +45,12 @@ RDOC_OPTS = ['--quiet', '--title', 'gigantron documentation',
     "--main", "README",
     "--inline-source"]
 
+
 class Hoe
+  silence_warnings do
+    RUBY_FLAGS = "-I#{%w(lib ext bin test).join(File::PATH_SEPARATOR)}" +
+      (ENV['RUBY_DEBUG'] ? " #{ENV['RUBY_DEBUG']}" : '')
+  end
   def extra_deps
     @extra_deps.reject! { |x| Array(x).first == 'hoe' }
     @extra_deps

data/config/requirements.rb

@@ -2,7 +2,7 @@ require 'fileutils'
 include FileUtils
 
 require 'rubygems'
-%w[rake hoe newgem rubigen].each do |req_gem|
+%w[rake hoe newgem rubigen activerecord activesupport shoulda].each do |req_gem|
   begin
     require req_gem
   rescue LoadError

data/gigantron_generators/mapreduce_task/USAGE

@@ -0,0 +1,5 @@
+Description:
+  
+  
+Usage:
+  

data/gigantron_generators/mapreduce_task/mapreduce_task_generator.rb

@@ -0,0 +1,54 @@
+class MapreduceTaskGenerator < RubiGen::Base
+
+  default_options :author => nil
+
+  attr_reader :name
+
+  def initialize(runtime_args, runtime_options = {})
+    super
+    usage if args.empty?
+    @name = args.shift
+    extract_options
+  end
+
+  def manifest
+    record do |m|
+      # Ensure appropriate folder(s) exists
+      m.directory 'test/'
+      m.directory 'test/tasks/'
+      m.template "test/tasks/test_task.rb", "test/tasks/test_#{@name}.rb"
+      m.directory 'tasks/'
+      m.template "tasks/task.rake", "tasks/#{@name}.rake"
+      m.directory 'mapreduce/'
+      m.template "mapreduce/mr_task.rb", "mapreduce/mr_#{@name}.rb"
+
+    end
+  end
+
+  protected
+    def banner
+      <<-EOS
+Creates a ...
+
+USAGE: #{$0} #{spec.name} name
+EOS
+    end
+
+    def add_options!(opts)
+      # opts.separator ''
+      # opts.separator 'Options:'
+      # For each option below, place the default
+      # at the top of the file next to "default_options"
+      # opts.on("-a", "--author=\"Your Name\"", String,
+      #         "Some comment about this option",
+      #         "Default: none") { |options[:author]| }
+      # opts.on("-v", "--version", "Show the #{File.basename($0)} version number and quit.")
+    end
+
+    def extract_options
+      # for each option, extract it into a local variable (and create an "attr_reader :author" at the top)
+      # Templates can access these value via the attr_reader-generated methods, but not the
+      # raw instance variable value.
+      # @author = options[:author]
+    end
+end

data/gigantron_generators/mapreduce_task/templates/mapreduce/mr_task.rb

@@ -0,0 +1,22 @@
+class MapReduce<%= name.capitalize %>
+  include SkynetDebugger
+
+  def self.run
+    job = Skynet::Job.new(
+      :mappers => 2, #change to whatever is a good number
+      :reducers => 1, #same
+      :map_reduce_class => self,
+      :map_data => [] #acquire your data!
+    )
+    job.run
+  end
+
+  def self.map(profiles)
+    #map code
+  end
+
+  def self.reduce(pairs)
+    #reduce code
+  end
+end
+

data/gigantron_generators/mapreduce_task/templates/tasks/task.rake

@@ -0,0 +1,5 @@
+desc "Describe your MapReduce task"
+task :<%= name %> do
+  load 'mapreduce/mr_<%= name %>'
+  MapReduce<%= name.capitalize %>.run
+end

data/gigantron_generators/mapreduce_task/templates/test/tasks/test_task.rb

@@ -0,0 +1,22 @@
+require File.dirname(__FILE__) + '/../test_helper.rb'
+
+class Test<%= name.capitalize %> < Test::Unit::TestCase
+  def setup
+    get_db_conn(GTRON_ENV)
+    @rake = Rake::Application.new
+    Rake.application = @rake
+    load File.dirname(__FILE__) + '/../../tasks/<%= name %>.rake'
+  end
+
+  should "be true" do
+    # Testing rake is a bit different
+    # http://blog.nicksieger.com/articles/2007/06/11/test-your-rake-tasks
+    # Example:
+    #   @rake["task_name"].invoke
+    assert true
+  end
+
+  def teardown
+    Rake.application = nil
+  end
+end

data/gigantron_generators/migration/USAGE

@@ -0,0 +1,5 @@
+Description:
+  
+  
+Usage:
+  

data/gigantron_generators/migration/migration_generator.rb

@@ -0,0 +1,61 @@
+class MigrationGenerator < RubiGen::Base
+
+  default_options :author => nil
+
+  attr_reader :name
+
+  def initialize(runtime_args, runtime_options = {})
+    super
+    usage if args.empty?
+    @name = args.shift
+    extract_options
+  end
+
+  def manifest
+    record do |m|
+      # Ensure appropriate folder(s) exists
+      m.directory "db/"
+      m.directory "db/migrate/"
+      m.template  "db/migrate/migration.rb", 
+        "db/migrate/#{next_migration_num}_#{name.underscore}.rb"
+
+    end
+  end
+
+  protected
+    def banner
+      <<-EOS
+Creates a ...
+
+USAGE: #{$0} #{spec.name} name
+EOS
+    end
+
+    def add_options!(opts)
+      # opts.separator ''
+      # opts.separator 'Options:'
+      # For each option below, place the default
+      # at the top of the file next to "default_options"
+      # opts.on("-a", "--author=\"Your Name\"", String,
+      #         "Some comment about this option",
+      #         "Default: none") { |options[:author]| }
+      # opts.on("-v", "--version", "Show the #{File.basename($0)} version number and quit.")
+    end
+
+    def extract_options
+      # for each option, extract it into a local variable (and create an "attr_reader :author" at the top)
+      # Templates can access these value via the attr_reader-generated methods, but not the
+      # raw instance variable value.
+      # @author = options[:author]
+    end
+
+    def next_migration_num
+      #blegh, catalog existing migrations, find next
+      current = Dir.glob(
+        "#{@destination_root}/db/migrate/[0-9][0-9][0-9]_*.rb").map{|x|
+          /(\d{3})_.*\.rb/.match(x)[1].to_i
+      }.max
+      current ||= 0
+      "%03d" % current.succ
+    end
+end

data/gigantron_generators/migration/templates/db/migrate/migration.rb

@@ -0,0 +1,7 @@
+class <%= name.camelcase %> < ActiveRecord::Migration
+  def self.up
+  end
+
+  def self.down
+  end
+end

data/gigantron_generators/model/model_generator.rb

@@ -14,11 +14,12 @@ class ModelGenerator < RubiGen::Base
   def manifest
     record do |m|
       m.directory "models/"
-      m.template  "models/model.rb", "models/#{@name}.rb"
+      m.template  "models/model.rb", "models/#{@name.underscore}.rb"
       
       m.directory "test/"
       m.directory "test/models/"
-      m.template  "test/models/test_model.rb", "test/models/test_#{name}.rb"
+      m.template  "test/models/test_model.rb", 
+        "test/models/test_#{name.underscore}.rb"
     end
   end
 

data/gigantron_generators/model/templates/models/model.rb

@@ -1,6 +1,3 @@
-class <%= name.capitalize %>
-  include DataMapper::Resource
-
-  property :id, Integer, :serial => true
+class <%= name.capitalize %> < ActiveRecord::Base
 
 end

data/gigantron_generators/model/templates/test/models/test_model.rb

@@ -3,7 +3,7 @@ require File.dirname(__FILE__) + '/../test_helper.rb'
 class Test<%= name.capitalize %> < Test::Unit::TestCase
   def setup
     get_db_conn(GTRON_ENV)
-    DataMapper.auto_migrate!
+    Gigantron.migrate_dbs
   end
 
   #replace with real tests

data/gigantron_generators/task/task_generator.rb

@@ -14,10 +14,11 @@ class TaskGenerator < RubiGen::Base
   def manifest
     record do |m|
       m.directory "tasks/"
-      m.template  "tasks/task.rake", "tasks/#{@name}.rake"
+      m.template  "tasks/task.rake", "tasks/#{@name.underscore}.rake"
       m.directory "test/"
       m.directory "test/tasks/"
-      m.template  "test/tasks/test_task.rb", "test/tasks/test_#{@name}.rb"
+      m.template  "test/tasks/test_task.rb", 
+        "test/tasks/test_#{@name.underscore}.rb"
     end
   end
 

data/lib/gigantron/migrator.rb

@@ -0,0 +1,10 @@
+
+# simple function to handle migrating Gigantron databases
+module Gigantron
+  def self.migrate_dbs
+    ActiveRecord::Migration.verbose = 
+      ENV["VERBOSE"] ? ENV["VERBOSE"] == "true" : true
+    ActiveRecord::Migrator.migrate("#{GTRON_ROOT}/db/migrate/", 
+      ENV["VERSION"] ? ENV["VERSION"].to_i : nil)
+  end
+end

data/lib/gigantron/tasks/db.rb

@@ -1,8 +1,11 @@
+require 'gigantron/migrator'
+
 namespace :db do
-  desc "Automatically migrate databases according to models"
-  task :automigrate do
+  desc "Migrate databases according to models"
+  task :migrate do
     puts "Migrating your database"
     get_db_conn(GTRON_ENV)
-    DataMapper.auto_migrate!
+    
+    Gigantron.migrate_dbs
   end
 end

data/lib/gigantron/version.rb

@@ -1,8 +1,8 @@
 module Gigantron #:nodoc:
   module VERSION #:nodoc:
     MAJOR = 0
-    MINOR = 0
-    TINY  = 1
+    MINOR = 1
+    TINY  = 0
 
     STRING = [MAJOR, MINOR, TINY].join('.')
   end

data/test/test_gigantron_generator.rb

@@ -11,21 +11,7 @@ class TestGigantronGenerator < Test::Unit::TestCase
     bare_teardown
   end
   
-  # Some generator-related assertions:
-  #   assert_generated_file(name, &block) # block passed the file contents
-  #   assert_directory_exists(name)
-  #   assert_generated_class(name, &block)
-  #   assert_generated_module(name, &block)
-  #   assert_generated_test_for(name, &block)
-  # The assert_generated_(class|module|test_for) &block is passed the body of the class/module within the file
-  #   assert_has_method(body, *methods) # check that the body has a list of methods (methods with parentheses not supported yet)
-  #
-  # Other helper methods are:
-  #   app_root_files - put this in teardown to show files generated by the test method (e.g. p app_root_files)
-  #   bare_setup - place this in setup method to create the APP_ROOT folder for each test
-  #   bare_teardown - place this in teardown method to destroy the TMP_ROOT or APP_ROOT folder after each test
-  
-  def test_generator_without_options
+  should "generate correct directory and file structure" do
     run_generator('gigantron', [APP_ROOT], sources)
     assert_directory_exists "tasks/"
     assert_generated_file   "tasks/import.rake"
@@ -43,14 +29,86 @@ class TestGigantronGenerator < Test::Unit::TestCase
     assert_generated_file   "Rakefile"
     assert_generated_file   "initialize.rb"
   end
+
+  context "Generated project" do
+    setup do
+      run_generator('gigantron', [APP_ROOT], sources)
+
+      silence_warnings { GTRON_ENV = :test }
+      ENV["GTRON_ENV"] = "test"
+
+      initialize_gigantron
+      configure_gigantron_db
+      get_db_conn(GTRON_ENV)
+    end
+
+    context "with migration and model" do
+      setup do
+        run_generator('model', ['Foo'], sources)
+        
+        create_test_migration
+        get_db_conn(GTRON_ENV)
+
+        ENV["VERBOSE"] = "false"
+        Gigantron.migrate_dbs
+      end
+
+      should "create test db" do
+        assert File.exists?("#{APP_ROOT}/db/test.sqlite3")
+      end
+
+      should "create table foos and populate" do
+        assert_equal Foo.find(:all).size, 2
+      end
+    end
+
+    context "with Rake tasks" do
+      setup do
+        @rake = Rake::Application.new
+        Rake.application = @rake
+        load "#{APP_ROOT}/tasks/import.rake"
+      end
+
+      teardown { Rake.application = nil }
+
+      should "run empty import task" do
+        assert @rake['import'].invoke
+      end
+    end
+  end
   
   private
   def sources
-    [RubiGen::PathSource.new(:test, File.join(File.dirname(__FILE__),"..", generator_path))
-    ]
+    [RubiGen::PathSource.new(:test, File.join(File.dirname(__FILE__),"..", generator_path)),
+    RubiGen::PathSource.new(:test_components, File.join(File.dirname(__FILE__),"..", "gigantron_generators"))]
   end
   
   def generator_path
     "app_generators"
   end
+
+  def initialize_gigantron
+    require File.join(APP_ROOT, "initialize.rb")
+    require 'gigantron/migrator'
+  end
+  
+  def configure_gigantron_db
+    if !File.exists? "test/template_database.yml"
+      flunk "No Database Configuration. Configure test DB in 'test/template_database.yml'!"
+    end
+    File.open("#{APP_ROOT}/database.yml", "w") do |f|
+      f.write ERB.new(
+        File.read("test/template_database.yml")).result(binding)
+    end
+  end
+
+  def create_test_migration
+    if !File.exists? "#{APP_ROOT}/db/migrate/001_create_foos.rb"
+      run_generator('migration', ['CreateFoos'], sources)
+    end
+ 
+    assert File.exists?("#{APP_ROOT}/db/migrate/001_create_foos.rb")
+    FileUtils.cp("test/template_migration.rb", 
+      "#{APP_ROOT}/db/migrate/001_create_foos.rb")
+  end
 end

data/test/test_mapreduce_task_generator.rb

@@ -0,0 +1,50 @@
+require File.join(File.dirname(__FILE__), "test_generator_helper.rb")
+
+
+class TestMapreduceTaskGenerator < Test::Unit::TestCase
+  include RubiGen::GeneratorTestHelper
+
+  def setup
+    bare_setup
+  end
+
+  def teardown
+    bare_teardown
+  end
+
+  # Some generator-related assertions:
+  #   assert_generated_file(name, &block) # block passed the file contents
+  #   assert_directory_exists(name)
+  #   assert_generated_class(name, &block)
+  #   assert_generated_module(name, &block)
+  #   assert_generated_test_for(name, &block)
+  # The assert_generated_(class|module|test_for) &block is passed the body of the class/module within the file
+  #   assert_has_method(body, *methods) # check that the body has a list of methods (methods with parentheses not supported yet)
+  #
+  # Other helper methods are:
+  #   app_root_files - put this in teardown to show files generated by the test method (e.g. p app_root_files)
+  #   bare_setup - place this in setup method to create the APP_ROOT folder for each test
+  #   bare_teardown - place this in teardown method to destroy the TMP_ROOT or APP_ROOT folder after each test
+
+  def test_generator_without_options
+    name = "myapp"
+    run_generator('mapreduce_task', [name], sources)
+    assert_directory_exists "mapreduce/"
+    assert_generated_file   "mapreduce/mr_#{name}.rb"
+    assert_directory_exists "tasks/"
+    assert_generated_file   "tasks/#{name}.rake"
+    assert_directory_exists "test/"
+    assert_directory_exists "test/tasks/"
+    assert_generated_file   "test/tasks/test_#{name}.rb"
+  end
+
+  private
+  def sources
+    [RubiGen::PathSource.new(:test, File.join(File.dirname(__FILE__),"..", generator_path))
+    ]
+  end
+
+  def generator_path
+    "gigantron_generators"
+  end
+end

data/test/test_migration_generator.rb

@@ -0,0 +1,49 @@
+require File.join(File.dirname(__FILE__), "test_generator_helper.rb")
+
+
+class TestMigrationGenerator < Test::Unit::TestCase
+  include RubiGen::GeneratorTestHelper
+
+  def setup
+    bare_setup
+  end
+
+  def teardown
+    bare_teardown
+  end
+
+  # Some generator-related assertions:
+  #   assert_generated_file(name, &block) # block passed the file contents
+  #   assert_directory_exists(name)
+  #   assert_generated_class(name, &block)
+  #   assert_generated_module(name, &block)
+  #   assert_generated_test_for(name, &block)
+  # The assert_generated_(class|module|test_for) &block is passed the body of the class/module within the file
+  #   assert_has_method(body, *methods) # check that the body has a list of methods (methods with parentheses not supported yet)
+  #
+  # Other helper methods are:
+  #   app_root_files - put this in teardown to show files generated by the test method (e.g. p app_root_files)
+  #   bare_setup - place this in setup method to create the APP_ROOT folder for each test
+  #   bare_teardown - place this in teardown method to destroy the TMP_ROOT or APP_ROOT folder after each test
+
+  def test_generator_without_options
+    name = "CreateFoo"
+    run_generator('migration', [name], sources)
+    assert_directory_exists "db/"
+    assert_directory_exists "db/migrate/"
+    assert_generated_file   "db/migrate/001_create_foo.rb"
+    name2 = "AlterFoo"
+    run_generator('migration', [name2], sources)
+    assert_generated_file  "db/migrate/002_alter_foo.rb"
+  end
+
+  private
+  def sources
+    [RubiGen::PathSource.new(:test, File.join(File.dirname(__FILE__),"..", generator_path))
+    ]
+  end
+
+  def generator_path
+    "gigantron_generators"
+  end
+end

data/test/test_model_generator.rb

@@ -27,13 +27,13 @@ class TestModelGenerator < Test::Unit::TestCase
   #   bare_teardown - place this in teardown method to destroy the TMP_ROOT or APP_ROOT folder after each test
 
   def test_generator_without_options
-    name = "foo"
+    name = "FooBaz"
     run_generator('model', [name], sources)
     assert_directory_exists "models/"
-    assert_generated_file   "models/#{name}.rb"
+    assert_generated_file   "models/foo_baz.rb"
     assert_directory_exists "test/"
     assert_directory_exists "test/models/"
-    assert_generated_file   "test/models/test_#{name}.rb"
+    assert_generated_file   "test/models/test_foo_baz.rb"
   end
 
   private

data/test/test_task_generator.rb

@@ -27,13 +27,13 @@ class TestTaskGenerator < Test::Unit::TestCase
   #   bare_teardown - place this in teardown method to destroy the TMP_ROOT or APP_ROOT folder after each test
 
   def test_generator_without_options
-    name = "bar"
+    name = "FooBar"
     run_generator('task', [name], sources)
     assert_directory_exists "tasks/"
-    assert_generated_file   "tasks/#{name}.rake"
+    assert_generated_file   "tasks/foo_bar.rake"
     assert_directory_exists "test/"
     assert_directory_exists "test/tasks"
-    assert_generated_file   "test/tasks/test_#{name}.rb"
+    assert_generated_file   "test/tasks/test_foo_bar.rb"
   end
 
   private

data/website/index.html

@@ -33,7 +33,7 @@
     <h1>Gigantron: Processor of Data</h1>
     <div id="version" class="clickable" onclick='document.location = "http://rubyforge.org/projects/gigantron"; return false'>
       <p>Get Version</p>
-      <a href="http://rubyforge.org/projects/gigantron" class="numbers">0.0.1</a>
+      <a href="http://rubyforge.org/projects/gigantron" class="numbers">0.1.0</a>
     </div>
     <h1>&#x2192; &#8216;gigantron&#8217;</h1>
 
@@ -43,7 +43,8 @@
 
 	<p>Gigantron is a simple framework for the creation and organization of
 data processing projects. Data-processing transforms are created as Rake tasks
-and data is handled through DataMapper models.</p>
+and data is handled through ActiveRecord models. (DataMapper was the original
+plan, but it has problems playing nicely with JRuby for now).</p>
 
 
 	<p>Ruby is great for exploratory data processing.  Data processing projects tend 
@@ -110,6 +111,67 @@ shell&gt; $ script/generate task modis_to_kml
 ActiveSupport for convenience.</p>
 
 
+	<h2>Hacking</h2>
+
+
+	<p>Gigantron is super minimal now, so modifying it is pretty easy.  The gigantron
+application generator (that which is invoked by the <code>gigantron</code> command) lives
+in <code>app_generators/gigantron/</code>.  The template files and the template dir
+structure are in <code>app_generators/gigantron/templates/</code>.  When adding new
+templates, directories, or files, add the names first to the tests in 
+<code>test/test_gigantron_generator.rb</code>, then the stubs to
+<code>app_generators/gigantron/templates</code>, and then finally describe them in the
+manifest section of <code>app_generators/gigantron/gigantron_generator.rb</code>.  It
+should look something like</p>
+
+
+	<p><pre class='syntax'>
+  <span class="keyword">def </span><span class="method">manifest</span>
+    <span class="ident">record</span> <span class="keyword">do</span> <span class="punct">|</span><span class="ident">m</span><span class="punct">|</span>
+      <span class="punct">...</span>
+      <span class="ident">m</span><span class="punct">.</span><span class="ident">file</span> <span class="punct">&quot;</span><span class="string">new_file</span><span class="punct">&quot;,</span> <span class="punct">&quot;</span><span class="string">new_file</span><span class="punct">&quot;</span> <span class="comment">#straight file copy</span>
+      <span class="ident">m</span><span class="punct">.</span><span class="ident">directory</span> <span class="punct">&quot;</span><span class="string">my_new_dir</span><span class="punct">&quot;</span> <span class="comment">#create directory</span>
+      <span class="ident">m</span><span class="punct">.</span><span class="ident">template</span> <span class="punct">&quot;</span><span class="string">new_thing</span><span class="punct">&quot;,</span> <span class="punct">&quot;</span><span class="string">new_thing</span><span class="punct">&quot;</span> <span class="comment">#runs file through ERB when copying</span>
+    <span class="keyword">end</span>
+  <span class="keyword">end</span>
+</pre></p>
+
+
+	<p>It might be handy to know that in <code>gigantron_generator.rb</code> the name provided to the generator can be referenced as <code>@name</code>.  This can be used like</p>
+
+
+	<p><pre class='syntax'>
+<span class="ident">record</span> <span class="keyword">do</span> <span class="punct">|</span><span class="ident">m</span><span class="punct">|</span>
+  <span class="ident">m</span><span class="punct">.</span><span class="ident">file</span> <span class="punct">&quot;</span><span class="string">renamed_file</span><span class="punct">&quot;,</span> <span class="punct">&quot;</span><span class="string"><span class="expr">#{@name}</span>_file</span><span class="punct">&quot;</span>
+<span class="keyword">end</span>
+</pre></p>
+
+
+	<p>The same value is available to your templates as just <code>name</code>.  You can template
+a file like</p>
+
+
+	<p><pre class='syntax'>
+<span class="keyword">class </span><span class="class">Test</span><span class="punct">&lt;%=</span><span class="string"> name.capitalize %&gt; &lt; Test::Unit::TestCase
+  ...
+end<span class="normal">
+</span></span></pre></p>
+
+
+	<p>The same process applies to the model and task generator for gigantron
+projects.  These generators live in <code>gigantron_generators/</code>.  Modifying them
+is exactly the same as modifying the application generator.</p>
+
+
+	<p>All of this is pretty vanilla <a href="http://rubigen.rubyforge.org">RubiGen</a>, so if in doubt, check out the docos on that fine piece of work.</p>
+
+
+	<p>The only other place for code in Gigantron is is <code>lib/gigantron/tasks/</code> where a few boilerplate test and db tasks live.  I think I ripped the test tasks off of rails.</p>
+
+
+	<p>If you have any questions, do contact me.  I am interested in anything that will make Gigantron suck less and be useful to people.</p>
+
+
 	<h2>How to submit patches</h2>
 
 
@@ -124,6 +186,8 @@ ActiveSupport for convenience.</p>
 
 
 <pre>cd gigantron
+cp test/template_database.yml.example test/template_database.yml
+vim test/template_database.yml
 rake test
 rake install_gem</pre>
 
@@ -138,7 +202,7 @@ rake install_gem</pre>
 
 	<p>Comments are welcome. Send an email to <a href="mailto:ben@pixelmachine.org">Ben Hughes</a></p>
     <p class="coda">
-      <a href="ben@pixelmachine.org">Ben Hughes</a>, 1st June 2008<br>
+      <a href="ben@pixelmachine.org">Ben Hughes</a>, 18th June 2008<br>
       Theme extended from <a href="http://rb2js.rubyforge.org/">Paul Battley</a>
     </p>
 </div>

data/website/index.txt

@@ -7,7 +7,8 @@ h2. What
 
 Gigantron is a simple framework for the creation and organization of
 data processing projects. Data-processing transforms are created as Rake tasks
-and data is handled through DataMapper models.
+and data is handled through ActiveRecord models. (DataMapper was the original
+plan, but it has problems playing nicely with JRuby for now).
 
 Ruby is great for exploratory data processing.  Data processing projects tend 
 to grow up and encompass large numbers of random scripts and input files.  It 
@@ -67,6 +68,55 @@ shell> $ script/generate task modis_to_kml
 One can edit these files to add functionality.  Gigantron by default includes
 ActiveSupport for convenience.
 
+h2. Hacking
+
+Gigantron is super minimal now, so modifying it is pretty easy.  The gigantron
+application generator (that which is invoked by the @gigantron@ command) lives
+in @app_generators/gigantron/@.  The template files and the template dir
+structure are in @app_generators/gigantron/templates/@.  When adding new
+templates, directories, or files, add the names first to the tests in 
+@test/test_gigantron_generator.rb@, then the stubs to
+@app_generators/gigantron/templates@, and then finally describe them in the
+manifest section of @app_generators/gigantron/gigantron_generator.rb@.  It
+should look something like
+
+<pre syntax="ruby">
+  def manifest
+    record do |m|
+      ...
+      m.file "new_file", "new_file" #straight file copy
+      m.directory "my_new_dir" #create directory
+      m.template "new_thing", "new_thing" #runs file through ERB when copying
+    end
+  end
+</pre>
+
+It might be handy to know that in @gigantron_generator.rb@ the name provided to the generator can be referenced as @@name@.  This can be used like
+
+<pre syntax="ruby">
+record do |m|
+  m.file "renamed_file", "#{@name}_file"
+end
+</pre>
+
+The same value is available to your templates as just @name@.  You can template
+a file like
+
+<pre syntax="ruby">
+class Test<%%= name.capitalize %> < Test::Unit::TestCase
+  ...
+end
+</pre>
+
+The same process applies to the model and task generator for gigantron
+projects.  These generators live in @gigantron_generators/@.  Modifying them
+is exactly the same as modifying the application generator.
+
+All of this is pretty vanilla "RubiGen":http://rubigen.rubyforge.org, so if in doubt, check out the docos on that fine piece of work.
+
+The only other place for code in Gigantron is is @lib/gigantron/tasks/@ where a few boilerplate test and db tasks live.  I think I ripped the test tasks off of rails.
+
+If you have any questions, do contact me.  I am interested in anything that will make Gigantron suck less and be useful to people.
 
 h2. How to submit patches
 
@@ -79,6 +129,8 @@ h2. How to submit patches
 h3. Build and test instructions
 
 <pre>cd gigantron
+cp test/template_database.yml.example test/template_database.yml
+vim test/template_database.yml
 rake test
 rake install_gem</pre>
 

metadata

@@ -1,39 +1,39 @@
---- !ruby/object:Gem::Specification 
-rubygems_version: 0.9.4
-specification_version: 1
-name: gigantron
-version: !ruby/object:Gem::Version 
-  version: 0.0.1
-date: 2008-06-01 00:00:00 -04:00
-summary: description of gem
-require_paths: 
-- lib
-email: 
-- ben@pixelmachine.org
-homepage: http://gigantron.rubyforge.org
-rubyforge_project: gigantron
-description: description of gem
-autorequire: 
-default_executable: 
-bindir: bin
-has_rdoc: true
-required_ruby_version: !ruby/object:Gem::Version::Requirement 
+--- !ruby/object:Gem::Specification
+required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
-  - - ">"
-    - !ruby/object:Gem::Version 
-      version: 0.0.0
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: !str 0
   version: 
-platform: ruby
-signing_key: 
-cert_chain: 
+email: 
+- ben@pixelmachine.org
+cert_chain: []
+summary: description of gem
 post_install_message: |
   
   For more information on gigantron, see http://gigantron.rubyforge.org
-  
-  Do you enjoy the tangy zip of miracle whip?
 
-authors: 
-- Ben Hughes
+  Do you enjoy the tangy zip of miracle whip?
+extra_rdoc_files: 
+- History.txt
+- License.txt
+- Manifest.txt
+- PostInstall.txt
+- README.txt
+- website/index.txt
+homepage: http://gigantron.rubyforge.org
+signing_key: 
+name: gigantron
+rdoc_options: 
+- --main
+- README.txt
+autorequire: 
+rubyforge_project: gigantron
+executables: 
+- gigantron
+description: description of gem
+specification_version: 2
+default_executable: 
 files: 
 - History.txt
 - License.txt
@@ -52,6 +52,14 @@ files:
 - bin/gigantron
 - config/hoe.rb
 - config/requirements.rb
+- gigantron_generators/mapreduce_task/USAGE
+- gigantron_generators/mapreduce_task/mapreduce_task_generator.rb
+- gigantron_generators/mapreduce_task/templates/mapreduce/mr_task.rb
+- gigantron_generators/mapreduce_task/templates/tasks/task.rake
+- gigantron_generators/mapreduce_task/templates/test/tasks/test_task.rb
+- gigantron_generators/migration/USAGE
+- gigantron_generators/migration/migration_generator.rb
+- gigantron_generators/migration/templates/db/migrate/migration.rb
 - gigantron_generators/model/USAGE
 - gigantron_generators/model/model_generator.rb
 - gigantron_generators/model/templates/models/model.rb
@@ -61,6 +69,7 @@ files:
 - gigantron_generators/task/templates/tasks/task.rake
 - gigantron_generators/task/templates/test/tasks/test_task.rb
 - lib/gigantron.rb
+- lib/gigantron/migrator.rb
 - lib/gigantron/tasks/db.rb
 - lib/gigantron/tasks/test.rb
 - lib/gigantron/version.rb
@@ -76,6 +85,8 @@ files:
 - test/test_gigantron.rb
 - test/test_gigantron_generator.rb
 - test/test_helper.rb
+- test/test_mapreduce_task_generator.rb
+- test/test_migration_generator.rb
 - test/test_model_generator.rb
 - test/test_task_generator.rb
 - website/index.html
@@ -83,28 +94,32 @@ files:
 - website/javascripts/rounded_corners_lite.inc.js
 - website/stylesheets/screen.css
 - website/template.html.erb
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements: 
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: !str 0
+  version: 
+extensions: []
+rubygems_version: 1.1.1
+requirements: []
+authors: 
+- Ben Hughes
+date: 2008-06-18 04:00:00 +00:00
+platform: ruby
 test_files: 
-- test/test_generator_helper.rb
-- test/test_gigantron.rb
-- test/test_task_generator.rb
-- test/test_gigantron_generator.rb
 - test/test_model_generator.rb
+- test/test_migration_generator.rb
 - test/test_helper.rb
-rdoc_options: 
-- --main
-- README.txt
-extra_rdoc_files: 
-- History.txt
-- License.txt
-- Manifest.txt
-- PostInstall.txt
-- README.txt
-- website/index.txt
-executables: 
-- gigantron
-extensions: []
-
-requirements: []
-
+- test/test_task_generator.rb
+- test/test_gigantron_generator.rb
+- test/test_mapreduce_task_generator.rb
+- test/test_gigantron.rb
+- test/test_generator_helper.rb
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+require_paths: 
+- lib
 dependencies: []
-
+bindir: bin
+has_rdoc: true