activewarehouse 0.1.0 → 0.2.0

data/README

@@ -1,32 +1,58 @@
-ActiveWarehouse
-===============
-The ActiveWarehouse library provides classes and functions which help with building Data Warehouses using Rails. It can be installed either as a plugin or as a Gem. If you install it as a gem you must include the following in your config/environment.rb:
+== ActiveWarehouse
+
+The ActiveWarehouse library provides classes and functions which help with building Data Warehouses using Rails. It can be installed either as a plugin or as a Gem.
+
+To install as a plugin just use:
+
+  script/plugin install --force svn://rubyforge.org/var/svn/activewarehouse/activewarehouse/trunk
+  
+To get the latest edge version.
+
+To install as a Gem, use:
+
+  gem install activewarehouse
+  
+At this point you can use ActiveWarehouse in any application using:
 
   require_gem 'activewarehouse'
   require 'active_warehouse'
   
-Additionally, to use the Rake tasks you must include the following in your Rakefile:
+If you want to use the Gem and link to your Rails app then use the gemsonrails project:
+
+  gem install gemsonrails
+
+And then in your Rails app:
 
-  load 'tasks/active_warehouse_tasks.rake'
+  rake gems:link GEM=activewarehouse
+
+== Generators
 
-Generators
-===============
 ActiveWarehouse comes with several generators
 
- script/generate fact Sales
- script/generate fact sales
+  script/generate fact Sales
+  script/generate fact sales
  
-   Creates a SalesFact class and a sales_facts table.
+    Creates a SalesFact class and a sales_facts table.
  
- script/generate dimension Region
- script/generate dimension region
+  script/generate dimension Region
+  script/generate dimension region
    
-   Creates a RegionDimension class and a region_dimension table.
+    Creates a RegionDimension class and a region_dimension table.
  
- script/generate cube RegionalSales
- script/generate cube regional_sales
+  script/generate cube RegionalSales
+  script/generate cube regional_sales
  
-   Creates a RegionalSalesCube class.
+    Creates a RegionalSalesCube class.
+   
+  script/generate bridge CustomerHierarchy
+  script/generate bridge customer_hierarchy
+   
+    Creates a CustomerHierarchyBridge class.
+   
+  script/generate dimension_view OrderDate Date
+  script/generate dimension_view order_date date
+  
+    Creates an OrderDateDimension class which is represented by a view on top of the DateDimension.
    
 The rules for naming are as follows:
 
@@ -38,4 +64,23 @@ Dimensions:
   Both the class name and the table name are suffixed by "_dimension".
 Cube:
   Cube class is singular. If a cube table is created it will also be singular.
-  
+Bridge:
+  Bridge classes and tables are both singular.
+  Both the class name and the table name are suffixed by "_bridge".
+Dimension View:
+  Dimension View classes are singular. The underlying data structure is a view on top of an existing dimension.
+  Both the class name and the view name are suffixed by "_dimension"
+  
+== ETL
+
+The ActiveWarehouse plugin does not directly handle Extract-Transform-Load processes, however the ActiveWarehouse ETL gem (installed separately) can help. To install it use:
+
+  gem install activewarehouse-etl
+  
+More information on the ETL process can be found at http://activewarehouse.rubyforge.org/etl
+
+== Tutorial
+
+A tutorial for ActiveWarehouse is available online at http://anthonyeden.com/2006/12/20/activewarehouse-example-with-rails-svn-logs
+
+You can also get a demo from the ActiveWarehouse subversion repository. Look in the SVN_ROOT/demo directory.

data/Rakefile

@@ -27,6 +27,17 @@ Rake::TestTask.new(:test) do |t|
   t.verbose = true
 end
 
+namespace :test do
+  desc 'Measures test coverage'
+  task :coverage do
+    rm_f "coverage"
+    rm_f "coverage.data"
+    rcov = "rcov --aggregate coverage.data --text-summary -Ilib"
+    system("#{rcov} test/**/*_test.rb")
+    system("open coverage/index.html") if PLATFORM['darwin']
+  end
+end
+
 desc 'Generate documentation for the active_warehouse plugin.'
 Rake::RDocTask.new(:rdoc) do |rdoc|
   rdoc.rdoc_dir = 'rdoc'
@@ -61,6 +72,7 @@ spec = Gem::Specification.new do |s|
   s.add_dependency('activesupport',    '>= 1.3.1.5618')
   s.add_dependency('activerecord',     '>= 1.14.4.5618')
   s.add_dependency('actionpack',       '>= 1.12.5.5618')
+  s.add_dependency('rails_sql_views',  '>= 0.1.0')
 
   s.rdoc_options << '--exclude' << '.'
   s.has_rdoc = false
@@ -118,4 +130,9 @@ task :release => [ :package ] do
     puts release_command
     system(release_command)
   end
+end
+
+desc "Publish the API documentation"
+task :pdoc => [:rdoc] do 
+  Rake::SshDirPublisher.new("aeden@rubyforge.org", "/var/www/gforge-projects/activewarehouse/rdoc", "rdoc").upload
 end

data/generators/bridge/USAGE

@@ -0,0 +1 @@
+./script/generate bridge NAME

data/generators/bridge/bridge_generator.rb

@@ -0,0 +1,46 @@
+class BridgeGenerator < Rails::Generator::NamedBase
+  attr_accessor :file_name
+  
+  default_options :skip_migration => false
+  
+  def initialize(runtime_args, runtime_options = {})
+    super
+    
+    @name = @name.tableize.singularize
+    @table_name = "#{@name}_bridge"
+    @class_name = "#{@name.camelize}Bridge"
+    @file_name = "#{@class_name.tableize.singularize}"
+  end
+  
+  def manifest
+    record do |m|
+      # Check for class naming collisions.
+      m.class_collisions class_path, "#{class_name}", "#{class_name}Test"
+      
+      # Create required directories if necessary
+      m.directory File.join('app/models', class_path)
+      m.directory File.join('test/unit', class_path)
+      m.directory File.join('test/fixtures', class_path)
+      
+      # Generate the files
+      m.template 'model.rb', File.join('app/models', class_path, "#{file_name}.rb")
+      m.template 'unit_test.rb', File.join('test/unit', class_path, "#{file_name}_test.rb")
+      m.template 'fixture.yml', File.join('test/fixtures', class_path, "#{table_name}.yml")
+      
+      # Generate the migration unless :skip_migration option is specified
+      unless options[:skip_migration]
+        m.migration_template 'migration.rb', 'db/migrate', :assigns => {
+          :migration_name => "Create#{class_name.gsub(/::/, '')}"
+        }, :migration_file_name => "create_#{file_name.gsub(/\//, '_')}"
+      end
+    end
+  end
+  
+  protected
+    def add_options!(opt)
+      opt.separator ''
+      opt.separator 'Options:'
+      opt.on("--skip-migration", 
+             "Don't generate a migration file for this bridge") { |v| options[:skip_migration] = v }
+    end
+end

data/generators/bridge/templates/fixture.yml

@@ -0,0 +1,5 @@
+# Read about fixtures at http://ar.rubyonrails.org/classes/Fixtures.html
+first:
+  id: 1
+another:
+  id: 2

data/generators/bridge/templates/migration.rb

@@ -0,0 +1,20 @@
+class <%= migration_name %> < ActiveRecord::Migration
+  def self.up
+    fields = {
+      # Add bridge attributes here as name => type
+      # Example: :top_flag => :integer
+    }
+    create_table :<%= table_name %> do |t|
+      fields.each do |name,type|
+        t.column name, type
+      end
+    end
+    fields.each do |name,type|
+      add_index :<%= table_name %>, name unless type == :text      
+    end
+  end
+
+  def self.down
+    drop_table :<%= table_name %>
+  end
+end

data/generators/bridge/templates/model.rb

@@ -0,0 +1,3 @@
+class <%= class_name %> < ActiveWarehouse::Bridge
+  
+end

data/generators/dimension/templates/unit_test.rb

@@ -1,8 +1,6 @@
 require File.dirname(__FILE__) + '<%= '/..' * class_nesting_depth %>/../test_helper'
 
 class <%= class_name %>Test < Test::Unit::TestCase
-  fixtures :<%= table_name %>
-
   # Replace this with your real tests.
   def test_truth
     assert true

data/generators/dimension_view/USAGE

@@ -0,0 +1 @@
+./script/generate dimension_view NAME

data/generators/dimension_view/dimension_view_generator.rb

@@ -0,0 +1,62 @@
+class DimensionViewGenerator < Rails::Generator::NamedBase
+  attr_accessor :file_name, :view_name, :query_target_name, :query_target_table_name, :view_query, :view_attributes
+  
+  default_options :skip_migration => false
+  
+  def initialize(runtime_args, runtime_options = {})
+    super
+    
+    usage if runtime_args.length != 2
+    
+    @name = @name.tableize.singularize
+    @query_target_name = runtime_args[1]
+    # define the view and query target table name
+    @view_name = "#{@name}_dimension"
+    @query_target_table_name = "#{query_target_name}_dimension"
+    # define the view class name and query target class name
+    @class_name = "#{view_name.camelize}"
+    @query_target_class_name = "#{query_target_table_name.camelize}"
+    # define the output file name
+    @file_name = "#{class_name.tableize.singularize}"
+    # define the query target class and expose its columns as attributes for the view
+    @query_target_class = @query_target_class_name.constantize
+    @view_attributes = @query_target_class.column_names
+    @view_query = "select * from #{query_target_table_name}"
+  end
+  
+  def manifest
+    record do |m|
+      # Check for class naming collisions.
+      m.class_collisions class_path, "#{class_name}", "#{class_name}Test"
+      
+      # Create required directories if necessary
+      m.directory File.join('app/models', class_path)
+      m.directory File.join('test/unit', class_path)
+      m.directory File.join('test/fixtures', class_path)
+      
+      # Generate the files
+      m.template 'model.rb', File.join('app/models', class_path, "#{file_name}.rb")
+      m.template 'unit_test.rb', File.join('test/unit', class_path, "#{file_name}_test.rb")
+      m.template 'fixture.yml', File.join('test/fixtures', class_path, "#{table_name}.yml")
+      
+      # Generate the migration unless :skip_migration option is specified
+      unless options[:skip_migration]
+        m.migration_template 'migration.rb', 'db/migrate', :assigns => {
+          :migration_name => "Create#{class_name.gsub(/::/, '')}"
+        }, :migration_file_name => "create_#{file_name.gsub(/\//, '_')}"
+      end
+    end
+  end
+  
+  def banner
+    "Usage: #{$0} #{spec.name} #{spec.name.camelize}Name SelectTarget [options]"
+  end
+  
+  protected
+    def add_options!(opt)
+      opt.separator ''
+      opt.separator 'Options:'
+      opt.on("--skip-migration", 
+             "Don't generate a migration file for this dimension view") { |v| options[:skip_migration] = v }
+    end
+end

data/generators/dimension_view/templates/migration.rb

@@ -0,0 +1,11 @@
+class <%= migration_name %> < ActiveRecord::Migration
+  def self.up
+    create_view "<%= view_name %>", <%= view_query do |t|
+      <%= view_attributes %>
+    end
+  end
+
+  def self.down
+    drop_view "<%= view_name %>"
+  end
+end

data/generators/dimension_view/templates/model.rb

@@ -0,0 +1,3 @@
+class <%= class_name %> < ActiveWarehouse::DimensionView
+  
+end

data/generators/dimension_view/templates/unit_test.rb

@@ -0,0 +1,10 @@
+require File.dirname(__FILE__) + '<%= '/..' * class_nesting_depth %>/../test_helper'
+
+class <%= class_name %>Test < Test::Unit::TestCase
+  fixtures :<%= view_name %>
+
+  # Replace this with your real tests.
+  def test_truth
+    assert true
+  end
+end

data/init.rb

@@ -0,0 +1 @@
+require 'active_warehouse'

data/lib/active_warehouse.rb

@@ -1,5 +1,9 @@
+# This source file requires all of the necessary gems and source files for ActiveWarehouse. If you
+# load this source file all of the other required files and gems will also be brought into the 
+# runtime.
+
 #--
-# Copyright (c) 2006 Anthony Eden
+# Copyright (c) 2006-2007 Anthony Eden
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
@@ -24,13 +28,17 @@
 $:.unshift(File.dirname(__FILE__)) unless
   $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
 
+require 'rubygems'
+unless Kernel.respond_to?(:gem)
+  Kernel.send :alias_method, :gem, :require_gem
+end
+
 unless defined?(ActiveSupport)
   begin
     $:.unshift(File.dirname(__FILE__) + "/../../activesupport/lib")  
     require 'active_support'  
   rescue LoadError
-    require 'rubygems'
-    require_gem 'activesupport'
+    gem 'activesupport'
   end
 end
 
@@ -39,8 +47,7 @@ unless defined?(ActiveRecord)
     $:.unshift(File.dirname(__FILE__) + "/../../activerecord/lib")
     require 'active_record'
   rescue LoadError
-    require 'rubygems'
-    require_gem 'activerecord'
+    gem 'activerecord'
   end
 end
 
@@ -51,15 +58,23 @@ unless defined?(ActionView)
     require 'action_controller'
     require 'action_view'
   rescue LoadError
-    require 'rubygems'
-    require_gem 'actionpack'
+    gem 'actionpack'
   end
 end
-  
+
+# Require 1.1.6 compatibility code if necessary
+require 'active_record/version'
+if ActiveRecord::VERSION::MAJOR < 1 || ActiveRecord::VERSION::MINOR < 15
+  require 'active_warehouse/compat/compat'
+end
 
 require 'active_warehouse/version'
 require 'active_warehouse/core_ext'
-require 'active_warehouse/model'
+require 'active_warehouse/aggregate'
+require 'active_warehouse/fact'
+require 'active_warehouse/dimension'
+require 'active_warehouse/cube'
+require 'active_warehouse/report'
 require 'active_warehouse/view'
 require 'active_warehouse/builder'
 require 'active_warehouse/migrations'

data/lib/active_warehouse/{model/aggregate.rb → aggregate.rb}

@@ -1,6 +1,6 @@
-module ActiveWarehouse
-  # An aggreate within a cube used to store calculated values
-  # Each aggregate will contain values for a dimension pair, down each of the dimension hierarchies
+module ActiveWarehouse #:nodoc:
+  # An aggreate within a cube used to store calculated values. Each aggregate will contain values for a dimension pair, 
+  # down each of the dimension hierarchies
   class Aggregate < ActiveRecord::Base
     class << self
       attr_accessor :name, :cube, :dimension1, :dimension2, :dimension1_hierarchy_name, :dimension2_hierarchy_name
@@ -68,6 +68,8 @@ module ActiveWarehouse
         #connection.execute("TRUNCATE TABLE #{table_name}") #TODO: make this generic to support all databases
         delete_all
         
+        $first = false
+        
         # aggregate the data for the two dimensions
         fact_class = cube.fact_class
         dim1 = Dimension.class_name(dimension1).constantize
@@ -81,29 +83,43 @@ module ActiveWarehouse
             
             stmt, fields = build_query(fact_class, dim1, dim1_stage_path, dim2, dim2_stage_path)
             
+            puts "\nSTMT: #{stmt}" if $first
+            
             # Get the facts and aggregate them
-            fact_class.connection.execute(stmt).each do |row|
+            # TODO: replace with select_all
+            fact_class.connection.select_all(stmt).each do |row|
+              require 'pp'
+              pp row if $first
               dim1_value = []
-              dim1_stage_path.each_with_index do |v, index|
-                dim1_value << row[index]
+              dim1_stage_path.each do |v|
+                dim1_value << row["#{v}"]
               end
               dim2_value = []
-              dim2_stage_path.each_with_index do |v, index|
-                dim2_value << row[dim1_stage_path.length + index]
+              dim2_stage_path.each do |v|
+                dim2_value << row["#{v}"]
               end
               
               agg_instance = new
               agg_instance.dimension1_path = dim1_value.join(':')
               agg_instance.dimension1_stage = dim1_stage_level
               agg_instance.dimension2_path = dim2_value.join(':')
-              agg_instance.dimension2_stage = dim2_stage_level 
-              fields.each_with_index do |field, index|
+              agg_instance.dimension2_stage = dim2_stage_level
+              
+              puts "DIM1_PATH: #{agg_instance.dimension1_path}" if $first
+              puts "DIM2_PATH: #{agg_instance.dimension2_path}" if $first
+              
+              
+              pp fields if $first
+              fields.each do |field|
                 # do the average here
-                agg_instance.send("#{field}=".to_sym, row[index + dim1_value.length + dim2_value.length])
+                puts "setting field #{field}, value is #{row[field.to_s]}" if $first
+                agg_instance.send("#{field}=".to_sym, row[field.to_s])
               end
               agg_instance.save!
               
               meta_data.update_attribute(:populated_at, Time.now)
+              
+              $first = false
             end
           end
         end
@@ -148,9 +164,9 @@ module ActiveWarehouse
         # put the SQL statement together
         stmt = "select #{fact_find_options[:select]} from "
         stmt << "#{fact_class.table_name} f #{fact_find_options[:joins]} "
-        stmt << "group by #{fact_find_options[:group]}"
+        stmt << "group by #{fact_find_options[:group]} "
 
-        return stmt, fields
+        return stmt.strip, fields
       end
       
     end