mainej-activewarehouse 0.3.0

data/activewarehouse/README

@@ -0,0 +1,99 @@
+== ActiveWarehouse
+
+The ActiveWarehouse library provides classes and functions which help with
+building Data Warehouses using Rails. It can be installed either as a plugin
+or as a Gem.
+
+To install as a plugin just use:
+
+  script/plugin install --force svn://rubyforge.org/var/svn/activewarehouse/activewarehouse/trunk
+  
+To get the latest edge version.
+
+To install as a Gem, use:
+
+  gem install activewarehouse
+
+On *nix you will need to run this command as root or better yet, using sudo.
+
+Next, you will need to freeze or link the Gem to your Rails app. I prefer using
+the gemsonrails project:
+
+  gem install gemsonrails
+
+And then in your Rails app:
+
+  rake gems:link GEM=activewarehouse
+  
+It is possible that freezing the Gem to the Rails app may not work at all times. It is most often best to install as a plugin.
+
+== Generators
+
+ActiveWarehouse comes with several generators
+
+  script/generate fact Sales
+  script/generate fact sales
+ 
+    Creates a SalesFact class and a sales_facts table.
+ 
+  script/generate dimension Region
+  script/generate dimension region
+   
+    Creates a RegionDimension class and a region_dimension table.
+ 
+  script/generate cube RegionalSales
+  script/generate cube regional_sales
+ 
+    Creates a RegionalSalesCube class.
+   
+  script/generate bridge CustomerHierarchy
+  script/generate bridge customer_hierarchy
+   
+    Creates a CustomerHierarchyBridge class.
+   
+  script/generate dimension_view OrderDate Date
+  script/generate dimension_view order_date date
+  
+    Creates an OrderDateDimension class which is represented by a view on top
+    of the DateDimension.
+   
+The rules for naming are as follows:
+
+Facts:
+  Fact classes and tables follow the typical Rails rules: classes are singular
+  and tables are pluralized. 
+  Both the class and table name are suffixed by "_fact".
+Dimensions:
+  Dimension classes and tables are both singular. 
+  Both the class name and the table name are suffixed by "_dimension".
+Cube:
+  Cube class is singular. If a cube table is created it will also be singular.
+Bridge:
+  Bridge classes and tables are both singular.
+  Both the class name and the table name are suffixed by "_bridge".
+Dimension View:
+  Dimension View classes are singular. The underlying data structure is a view
+  on top of an existing dimension.
+  Both the class name and the view name are suffixed by "_dimension"
+  
+== ETL
+
+The ActiveWarehouse plugin does not directly handle Extract-Transform-Load
+processes, however the ActiveWarehouse ETL gem (installed separately) can help.
+To install it use:
+
+  gem install activewarehouse-etl
+
+Once again you should run this command as root or using sudo.
+  
+More information on the ETL process can be found at
+http://activewarehouse.rubyforge.org/etl
+
+== Tutorial
+
+A tutorial for ActiveWarehouse is available online at
+http://anthonyeden.com/2006/12/20/activewarehouse-example-with-rails-svn-logs
+(Note that is is out of date.)
+
+You can also get a demo from the ActiveWarehouse subversion repository. Look in
+the SVN_ROOT/demo directory.

data/activewarehouse/Rakefile

@@ -0,0 +1,165 @@
+require 'rake'
+require 'rake/testtask'
+require 'rake/rdoctask'
+require 'rake/gempackagetask'
+require 'rake/contrib/rubyforgepublisher'
+require 'spec/version'
+require 'spec/rake/spectask'
+
+require File.join(File.dirname(__FILE__), 'lib/active_warehouse', 'version')
+
+module AW
+  PKG_BUILD       = ENV['PKG_BUILD'] ? '.' + ENV['PKG_BUILD'] : ''
+  PKG_NAME        = 'activewarehouse'
+  PKG_VERSION     = ActiveWarehouse::VERSION::STRING + PKG_BUILD
+  PKG_FILE_NAME   = "#{PKG_NAME}-#{PKG_VERSION}"
+  PKG_DESTINATION = ENV["PKG_DESTINATION"] || "../#{PKG_NAME}"
+end
+
+desc 'Default: run tests and specs.'
+task :default => [:test, :spec]
+
+desc 'Test the active_warehouse plugin.'
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'lib'
+  t.pattern = 'test/**/*_test.rb'
+  t.verbose = true
+end
+
+desc "Run all specs"
+Spec::Rake::SpecTask.new do |t|
+  t.spec_files = FileList['spec/**/*_spec.rb']
+  t.spec_opts = ['--options', 'spec/spec.opts']
+  unless ENV['NO_RCOV']
+    t.rcov = true
+    t.rcov_dir = '../doc/output/coverage'
+    t.rcov_opts = ['--exclude', 'spec\/spec,bin\/spec,examples,\/var\/lib\/gems,\/Library\/Ruby,\.autotest']
+  end
+end
+
+namespace :rcov do
+  desc 'Measures test coverage'
+  task :test do
+    rm_f 'coverage.data'
+    mkdir 'coverage' unless File.exist?('coverage')
+    rcov = "rcov --aggregate coverage.data --text-summary -Ilib"
+    system("#{rcov} test/*_test.rb")
+    system("open coverage/index.html") if PLATFORM['darwin']
+  end
+end
+
+desc 'Generate documentation for the active_warehouse plugin.'
+Rake::RDocTask.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'ActiveWarehouse'
+  rdoc.options << '--line-numbers' << '--inline-source'
+  rdoc.rdoc_files.include('README')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+# Gem Spec
+module AW
+  def self.package_files(package_prefix)
+    FileList[
+      "#{package_prefix}[a-zA-Z]*.rb",
+      "#{package_prefix}README",
+      "#{package_prefix}TODO",
+      "#{package_prefix}Rakefile",
+      "#{package_prefix}db/**/*",
+      "#{package_prefix}doc/**/*",
+      "#{package_prefix}generators/**/*", 
+      "#{package_prefix}lib/**/*",
+      "#{package_prefix}tasks/**/*"
+    ] - [ "#{package_prefix}test" ]
+  end
+
+  def self.spec(package_prefix = '')
+    Gem::Specification.new do |s|
+      s.name = 'activewarehouse'
+      s.version = AW::PKG_VERSION
+      s.summary = "Build data warehouses with Rails."
+      s.description = <<-EOF
+        ActiveWarehouse extends Rails to provide functionality specific for building data warehouses.
+      EOF
+
+      s.add_dependency('rake',                '>= 0.7.1')
+      s.add_dependency('fastercsv',           '>= 1.1.0')
+      s.add_dependency('activesupport',       '>= 1.3.1')
+      s.add_dependency('activerecord',        '>= 1.14.4')
+      s.add_dependency('actionpack',          '>= 1.12.5')
+      s.add_dependency('rails_sql_views',     '>= 0.1.0')
+      s.add_dependency('adapter_extensions',  '>= 0.1.0')
+
+      s.rdoc_options << '--exclude' << '.'
+      s.has_rdoc = false
+
+      s.files = package_files(package_prefix).to_a.delete_if {|f| f.include?('.svn')}
+      s.require_path = 'lib'
+
+      #s.bindir = "bin" # Use these for applications.
+      #s.executables = []
+      #s.default_executable = ""
+
+      s.author = "Anthony Eden"
+      s.email = "anthonyeden@gmail.com"
+      s.homepage = "http://activewarehouse.rubyforge.org"
+      s.rubyforge_project = "activewarehouse"
+    end
+  end
+end
+
+Rake::GemPackageTask.new(AW.spec) do |pkg|
+  pkg.gem_spec = AW.spec
+  pkg.need_tar = true
+  pkg.need_zip = true
+end
+
+desc "Generate code statistics"
+task :lines do
+  lines, codelines, total_lines, total_codelines = 0, 0, 0, 0
+
+  for file_name in FileList["lib/**/*.rb"]
+    next if file_name =~ /vendor/
+    f = File.open(file_name)
+
+    while line = f.gets
+      lines += 1
+      next if line =~ /^\s*$/
+      next if line =~ /^\s*#/
+      codelines += 1
+    end
+    puts "L: #{sprintf("%4d", lines)}, LOC #{sprintf("%4d", codelines)} | #{file_name}"
+
+    total_lines     += lines
+    total_codelines += codelines
+
+    lines, codelines = 0, 0
+  end
+
+  puts "Total: Lines #{total_lines}, LOC #{total_codelines}"
+end
+
+desc "Publish the release files to RubyForge."
+task :release => [ :package ] do
+  `rubyforge login`
+
+  for ext in %w( gem tgz zip )
+    release_command = "rubyforge add_release activewarehouse #{AW::PKG_NAME} 'REL #{AW::PKG_VERSION}' pkg/#{AW::PKG_NAME}-#{AW::PKG_VERSION}.#{ext}"
+    puts release_command
+    system(release_command)
+  end
+end
+
+desc "Publish the API documentation"
+task :pdoc => [:rdoc] do 
+  Rake::SshDirPublisher.new("aeden@rubyforge.org", "/var/www/gforge-projects/activewarehouse/rdoc", "rdoc").upload
+end
+
+desc "Reinstall the gem from a local package copy"
+task :reinstall => [:package] do
+  windows = RUBY_PLATFORM =~ /mswin/
+  sudo = windows ? '' : 'sudo'
+  gem = windows ? 'gem.bat' : 'gem'
+  `#{sudo} #{gem} uninstall -x -i #{AW::PKG_NAME}`
+  `#{sudo} #{gem} install pkg/#{AW::PKG_NAME}-#{AW::PKG_VERSION}`
+end

data/activewarehouse/TODO

@@ -0,0 +1,4 @@
+TODO list:
+
+* Modify the fact generator so that the migration has a map for foreign key and a map for fact attributes. The foreign keys can then automatically be indexed as that is pretty much standard behavior.
+* Make the belongs_to connection between facts and dimensions automatic.

data/activewarehouse/db/migrations/001_create_table_reports.rb

@@ -0,0 +1,28 @@
+class CreateTableReports < ActiveRecord::Migration
+  def self.up
+    create_table :table_reports do |t|
+      t.column :title, :string
+      t.column :cube_name, :string, :null => false
+      
+      t.column :column_dimension_name, :string
+      t.column :column_hierarchy, :string
+      t.column :column_constraints, :text
+      t.column :column_stage, :integer
+      t.column :column_param_prefix, :string
+      
+      t.column :row_dimension_name, :string
+      t.column :row_hierarchy, :string
+      t.column :row_constraints, :text
+      t.column :row_stage, :integer
+      t.column :row_param_prefix, :string
+      
+      t.column :fact_attributes, :text
+      
+      t.column :created_at, :datetime
+      t.column :updated_at, :datetime
+    end
+  end
+  def self.down
+    drop_table :table_reports
+  end
+end

data/activewarehouse/doc/references.txt

@@ -0,0 +1,4 @@
+The following papers are relevant to building, storing and querying data cubes with large databases.
+
+http://research.microsoft.com/research/pubs/view.aspx?msr_tr_id=MSR-TR-95-22
+http://dbpubs.stanford.edu/pub/showDoc.Fulltext?lang=en&doc=1995-34&format=pdf&compression=&name=1995-34.pdf

data/activewarehouse/generators/bridge/USAGE

@@ -0,0 +1 @@
+./script/generate bridge NAME

data/activewarehouse/generators/bridge/bridge_generator.rb

@@ -0,0 +1,46 @@
+class BridgeGenerator < Rails::Generator::NamedBase
+  attr_accessor :file_name
+  
+  default_options :skip_migration => false
+  
+  def initialize(runtime_args, runtime_options = {})
+    super
+    
+    @name = @name.tableize.singularize
+    @table_name = "#{@name}_bridge"
+    @class_name = "#{@name.camelize}Bridge"
+    @file_name = "#{@class_name.tableize.singularize}"
+  end
+  
+  def manifest
+    record do |m|
+      # Check for class naming collisions.
+      m.class_collisions class_path, "#{class_name}", "#{class_name}Test"
+      
+      # Create required directories if necessary
+      m.directory File.join('app/models', class_path)
+      m.directory File.join('test/unit', class_path)
+      m.directory File.join('test/fixtures', class_path)
+      
+      # Generate the files
+      m.template 'model.rb', File.join('app/models', class_path, "#{file_name}.rb")
+      m.template 'unit_test.rb', File.join('test/unit', class_path, "#{file_name}_test.rb")
+      m.template 'fixture.yml', File.join('test/fixtures', class_path, "#{table_name}.yml")
+      
+      # Generate the migration unless :skip_migration option is specified
+      unless options[:skip_migration]
+        m.migration_template 'migration.rb', 'db/migrate', :assigns => {
+          :migration_name => "Create#{class_name.gsub(/::/, '')}"
+        }, :migration_file_name => "create_#{file_name.gsub(/\//, '_')}"
+      end
+    end
+  end
+  
+  protected
+    def add_options!(opt)
+      opt.separator ''
+      opt.separator 'Options:'
+      opt.on("--skip-migration", 
+             "Don't generate a migration file for this bridge") { |v| options[:skip_migration] = v }
+    end
+end

data/activewarehouse/generators/bridge/templates/fixture.yml

@@ -0,0 +1,5 @@
+# Read about fixtures at http://ar.rubyonrails.org/classes/Fixtures.html
+first:
+  id: 1
+another:
+  id: 2

data/activewarehouse/generators/bridge/templates/migration.rb

@@ -0,0 +1,27 @@
+class <%= migration_name %> < ActiveRecord::Migration
+  def self.up
+    fields = {
+      # the following are the required bridge table columns for
+      # variable depth hierarchies.  Do not change them unless you know
+      # what you are doing.
+      :parent_id => :integer,
+      :child_id => :integer,
+      :num_levels_from_parent => :integer,
+      :is_bottom => :boolean,
+      :is_top => :boolean
+    }
+    create_table :<%= table_name %> do |t|
+      fields.each do |name,type|
+        t.column name, type
+      end
+    end
+    fields.each do |name,type|
+      add_index :<%= table_name %>, name unless type == :text      
+    end
+    add_index :<%= table_name %>, [:parent_id, :child_id, :num_levels_from_parent], :unique => true
+  end
+
+  def self.down
+    drop_table :<%= table_name %>
+  end
+end

data/activewarehouse/generators/bridge/templates/model.rb

@@ -0,0 +1,3 @@
+class <%= class_name %> < ActiveWarehouse::Bridge
+  
+end

data/activewarehouse/generators/bridge/templates/unit_test.rb

@@ -0,0 +1,8 @@
+require File.dirname(__FILE__) + '<%= '/..' * class_nesting_depth %>/../test_helper'
+
+class <%= class_name %>Test < Test::Unit::TestCase
+  # Replace this with your real tests.
+  def test_truth
+    assert true
+  end
+end

data/activewarehouse/generators/cube/USAGE

@@ -0,0 +1 @@
+./script/generate cube NAME

data/activewarehouse/generators/cube/cube_generator.rb

@@ -0,0 +1,28 @@
+class CubeGenerator < Rails::Generator::NamedBase
+  attr_accessor :file_name
+  
+  def initialize(runtime_args, runtime_options = {})
+    super
+    
+    @name = @name.underscore
+    @table_name = "#{@name}_cube"
+    @class_name = "#{@name.camelize}Cube"
+    @file_name = "#{@class_name.tableize.singularize}"
+  end
+  
+  def manifest
+    record do |m|
+      # Check for class naming collisions.
+      m.class_collisions class_path, "#{class_name}", "#{class_name}Test"
+      
+      # Create required directories if necessary
+      m.directory File.join('app/models', class_path)
+      m.directory File.join('test/unit', class_path)
+      
+      # Generate the files
+      m.template 'model.rb', File.join('app/models', class_path, "#{file_name}.rb")
+      m.template 'unit_test.rb', File.join('test/unit', class_path, "#{file_name}_test.rb")
+    end
+  end
+
+end

data/activewarehouse/generators/cube/templates/model.rb

@@ -0,0 +1,3 @@
+class <%= class_name %> < ActiveWarehouse::Cube
+  
+end

data/activewarehouse/generators/cube/templates/unit_test.rb

@@ -0,0 +1,8 @@
+require File.dirname(__FILE__) + '<%= '/..' * class_nesting_depth %>/../test_helper'
+
+class <%= class_name %>Test < Test::Unit::TestCase
+  # Replace this with your real tests.
+  def test_truth
+    assert true
+  end
+end