cranium 0.2.0

data/lib/cranium/dsl/database_definition.rb

@@ -0,0 +1,23 @@
+class Cranium::DSL::DatabaseDefinition
+
+  class << self
+    include Cranium::AttributeDSL
+  end
+
+  attr_reader :name
+
+  define_attribute :connect_to
+
+
+
+  def initialize(name)
+    @name = name
+  end
+
+
+
+  def ==(other)
+    name == other.name and connect_to == other.connect_to
+  end
+
+end

data/lib/cranium/dsl/extract_definition.rb

@@ -0,0 +1,28 @@
+class Cranium::DSL::ExtractDefinition
+
+  class << self
+    include Cranium::AttributeDSL
+  end
+
+  attr_reader :name, :storage
+
+  define_attribute :from
+  define_attribute :query
+  define_attribute :columns
+  define_attribute :incrementally_by
+
+
+
+  def initialize(name)
+    @name = name
+    @storage = Cranium::Extract::Storage.new name
+  end
+
+
+
+  def last_extracted_value_of(field, default = nil)
+    stored_value = @storage.last_value_of field
+    stored_value.nil? ? default : stored_value
+  end
+
+end

data/lib/cranium/dsl/import_definition.rb

@@ -0,0 +1,50 @@
+class Cranium::DSL::ImportDefinition
+
+  class << self
+    include Cranium::AttributeDSL
+  end
+
+
+  attr_reader :name
+  attr_reader :field_associations
+  attr_reader :merge_fields
+
+  define_attribute :into
+  define_boolean_attribute :truncate_insert
+  define_array_attribute :delete_insert_on
+
+
+  def initialize(name)
+    @name = name
+    @field_associations = {}
+    @merge_fields = {}
+  end
+
+
+
+  def put(fields)
+    @field_associations.merge! fields_hash(fields)
+  end
+
+
+
+  def merge_on(merge_fields)
+    @merge_fields = fields_hash(merge_fields)
+  end
+
+
+
+  private
+
+  def fields_hash(fields)
+    case fields
+      when Hash
+        return fields
+      when Symbol
+        return { fields => fields }
+      else
+        raise ArgumentError, "Unsupported argument for Import::#{caller[0][/`.*'/][1..-2]}"
+    end
+  end
+
+end

data/lib/cranium/dsl/source_definition.rb

@@ -0,0 +1,67 @@
+class Cranium::DSL::SourceDefinition
+
+  class << self
+    include Cranium::AttributeDSL
+  end
+
+  attr_reader :name
+  attr_reader :files
+  attr_reader :fields
+
+  define_attribute :file
+  define_attribute :delimiter
+  define_attribute :escape
+  define_attribute :quote
+  define_attribute :encoding
+
+
+
+  def initialize(name)
+    @name = name
+    @file = default_file_name
+    @fields = {}
+    @delimiter = ","
+    @escape = '"'
+    @quote = '"'
+    @encoding = "UTF-8"
+  end
+
+
+
+  def resolve_files
+    @files = Dir[File.join Cranium.configuration.upload_path, @file].map { |file| File.basename file }.sort
+  end
+
+
+
+  def field(name, type)
+    @fields[name] = type
+  end
+
+
+
+  def file_name_overriden?
+    @file != default_file_name
+  end
+
+
+
+  def ==(other)
+    name == other.name and
+      file == other.file and
+      delimiter == other.delimiter and
+      escape == other.escape and
+      quote == other.quote and
+      encoding == other.encoding and
+      fields == other.fields
+  end
+
+
+
+  private
+
+  def default_file_name
+    "#{@name}.csv"
+  end
+
+end

data/lib/cranium/dsl.rb

@@ -0,0 +1,100 @@
+module Cranium::DSL
+
+  autoload :DatabaseDefinition, 'cranium/dsl/database_definition'
+  autoload :ExtractDefinition, 'cranium/dsl/extract_definition'
+  autoload :ImportDefinition, 'cranium/dsl/import_definition'
+  autoload :SourceDefinition, 'cranium/dsl/source_definition'
+
+
+
+  def database(name, &block)
+    Cranium::Database.register_database name, &block
+  end
+
+
+
+  def source(name, &block)
+    Cranium.application.register_source name, &block
+  end
+
+
+
+  def extract(name, &block)
+    extract_definition = ExtractDefinition.new name
+    extract_definition.instance_eval &block
+    Cranium::Extract::DataExtractor.new.execute extract_definition
+  end
+
+
+
+  def read(name, &block)
+    source = Cranium.application.sources[name]
+    Cranium::DataReader.new(source).read(&block)
+  end
+
+
+
+  def transform(names, &block)
+    source = Cranium.application.sources[names.keys.first]
+    target = Cranium.application.sources[names.values.first]
+
+    Cranium::DataTransformer.new(source, target).transform(&block)
+  end
+
+
+
+  def deduplicate(source, options)
+    transform source => options[:into] do |record|
+      output record if unique_on_fields? *options[:by]
+    end
+  end
+
+
+
+  def join(source_name, options)
+    Cranium::Transformation::Join.new.tap do |join|
+      join.source_left = Cranium.application.sources[source_name]
+      join.source_right = Cranium.application.sources[options[:with]]
+      join.target = Cranium.application.sources[options[:into]]
+      join.match_fields = options[:match_on]
+      join.type = options[:type] || :inner
+    end.execute
+  end
+
+
+
+  def import(name, &block)
+    import_definition = ImportDefinition.new(name)
+    import_definition.instance_eval &block
+    Cranium::DataImporter.new.import import_definition
+  end
+
+
+
+  def archive(*sources)
+    sources.each do |source_name|
+      Cranium::Archiver.archive *Cranium.application.sources[source_name].files
+    end
+  end
+
+
+
+  def remove(*sources)
+    sources.each do |source_name|
+      Cranium::Archiver.remove *Cranium.application.sources[source_name].files
+    end
+  end
+
+
+
+  def sequence(name)
+    Cranium::Transformation::Sequence.new name
+  end
+
+
+
+  def after(&block)
+    Cranium.application.register_hook :after, &block
+  end
+
+end

data/lib/cranium/extensions/file.rb

@@ -0,0 +1,7 @@
+class File
+
+  def self.line_count(file_path)
+    (`wc -l #{file_path}`.match /^\s*(?<line_count>\d+).*/)["line_count"].to_i
+  end
+
+end

data/lib/cranium/extensions/sequel_greenplum.rb

@@ -0,0 +1,30 @@
+require 'sequel'
+Sequel.require 'adapters/shared/postgres'
+
+module Sequel::Postgres::DatasetMethods
+
+  def insert_clause_methods
+    remove_returning_from INSERT_CLAUSE_METHODS, :insert
+  end
+
+
+
+  def update_clause_methods
+    remove_returning_from UPDATE_CLAUSE_METHODS, :update
+  end
+
+
+
+  def delete_clause_methods
+    remove_returning_from DELETE_CLAUSE_METHODS, :delete
+  end
+
+
+
+  private
+
+  def remove_returning_from(methods, type)
+    methods.reject { |method| method == :"#{type}_returning_sql" }
+  end
+
+end

data/lib/cranium/external_table.rb

@@ -0,0 +1,75 @@
+class Cranium::ExternalTable
+
+  def initialize(source, db_connection)
+    @source, @connection = source, db_connection
+  end
+
+
+
+  def create
+    @connection.run <<-sql
+      CREATE EXTERNAL TABLE "#{name}" (
+          #{field_definitions}
+      )
+      LOCATION (#{external_location})
+      FORMAT 'CSV' (DELIMITER '#{quote @source.delimiter}' ESCAPE '#{quote @source.escape}' QUOTE '#{quote @source.quote}' HEADER)
+      ENCODING 'UTF8'
+    sql
+  end
+
+
+
+  def destroy
+    @connection.run %Q[DROP EXTERNAL TABLE "#{name}"]
+  end
+
+
+
+  def name
+    :"external_#{@source.name}"
+  end
+
+
+
+  private
+
+  def field_definitions
+    @source.fields.map do |name, type|
+      %Q("#{name}" #{sql_type_for_ruby_type(type)})
+    end.join ",\n          "
+  end
+
+
+
+  def sql_type_for_ruby_type(type)
+    case type.to_s
+      when "Integer" then
+        "INTEGER"
+      when "Float" then
+        "NUMERIC"
+      when "Date" then
+        "DATE"
+      when "Time" then
+        "TIMESTAMP WITHOUT TIME ZONE"
+      when "TrueClass", "FalseClass" then
+        "BOOLEAN"
+      else
+        "TEXT"
+    end
+  end
+
+
+
+  def quote(text)
+    text.gsub "'", "''"
+  end
+
+
+
+  def external_location
+    @source.files.map do |file_name|
+      "'gpfdist://#{Cranium.configuration.gpfdist_url}/#{Cranium.configuration.upload_directory}/#{file_name}'"
+    end.join(', ')
+  end
+
+end

data/lib/cranium/extract/data_extractor.rb

@@ -0,0 +1,11 @@
+class Cranium::Extract::DataExtractor
+
+  def execute(extract_definition)
+    if extract_definition.incrementally_by.nil?
+      Cranium::Extract::Strategy::Simple.new.execute extract_definition
+    else
+      Cranium::Extract::Strategy::Incremental.new.execute extract_definition
+    end
+  end
+
+end

data/lib/cranium/extract/storage.rb

@@ -0,0 +1,57 @@
+require 'yaml'
+require 'fileutils'
+
+class Cranium::Extract::Storage
+
+  STORAGE_FILE_NAME = "extracts"
+
+
+
+  def initialize(extract_name)
+    @extract_name = extract_name
+  end
+
+
+
+  def last_value_of(field)
+    stored_values[:last_values][field]
+  end
+
+
+
+  def save_last_value_of(field, value)
+    stored_values[:last_values][field] = value
+    save_stored_values
+  end
+
+
+
+  private
+
+  def stored_values
+    return @stored_values[@extract_name] unless @stored_values.nil?
+    @stored_values = (File.exists? storage_file) ? YAML.load(File.read storage_file) : {}
+    @stored_values[@extract_name] = { last_values: {} } if @stored_values[@extract_name].nil?
+    @stored_values[@extract_name]
+  end
+
+
+
+  def storage_file
+    File.join storage_dir, STORAGE_FILE_NAME
+  end
+
+
+
+  def storage_dir
+    Cranium.configuration.storage_directory
+  end
+
+
+
+  def save_stored_values
+    FileUtils.mkdir_p storage_dir unless Dir.exists? storage_dir
+    File.write storage_file, YAML.dump(@stored_values)
+  end
+
+end

data/lib/cranium/extract/strategy/base.rb

@@ -0,0 +1,27 @@
+require 'csv'
+
+class Cranium::Extract::Strategy::Base
+
+  def execute(extract_definition)
+    target_file_name = "#{extract_definition.name}.csv"
+    target_file_path = File.join Cranium.configuration.upload_path, target_file_name
+
+    raise StandardError, %Q(Extract halted: a file named "#{target_file_name}" already exists) if File.exists? target_file_path
+
+    CSV.open target_file_path, "w:UTF-8" do |target_file|
+      dataset = Cranium::Database[extract_definition.from].fetch extract_definition.query
+
+      target_file << (extract_definition.columns || dataset.columns)
+      write_dataset_into_file target_file, dataset, extract_definition
+    end
+  end
+
+
+
+  protected
+
+  def write_dataset_into_file(target_file, dataset, extract_definition)
+    raise "This template method must be overriden in descendants"
+  end
+
+end

data/lib/cranium/extract/strategy/incremental.rb

@@ -0,0 +1,16 @@
+class Cranium::Extract::Strategy::Incremental < Cranium::Extract::Strategy::Base
+
+  protected
+
+  def write_dataset_into_file(target_file, dataset, extract_definition)
+    incremental_field, max_value = extract_definition.incrementally_by, nil
+
+    dataset.each do |row|
+      max_value = row[incremental_field] if max_value.nil? or row[incremental_field] > max_value
+      target_file << row.values
+    end
+
+    extract_definition.storage.save_last_value_of incremental_field, max_value unless max_value.nil?
+  end
+
+end

data/lib/cranium/extract/strategy/simple.rb

@@ -0,0 +1,9 @@
+class Cranium::Extract::Strategy::Simple < Cranium::Extract::Strategy::Base
+
+  protected
+
+  def write_dataset_into_file(target_file, dataset, extract_definition)
+    dataset.each { |row| target_file << row.values }
+  end
+
+end

data/lib/cranium/extract/strategy.rb

@@ -0,0 +1,7 @@
+module Cranium::Extract::Strategy
+
+  autoload :Base, 'cranium/extract/strategy/base'
+  autoload :Incremental, 'cranium/extract/strategy/incremental'
+  autoload :Simple, 'cranium/extract/strategy/simple'
+
+end

data/lib/cranium/extract.rb

@@ -0,0 +1,7 @@
+module Cranium::Extract
+
+  autoload :DataExtractor, 'cranium/extract/data_extractor'
+  autoload :Storage, 'cranium/extract/storage'
+  autoload :Strategy, 'cranium/extract/strategy'
+
+end

data/lib/cranium/import_strategy/base.rb

@@ -0,0 +1,55 @@
+class Cranium::ImportStrategy::Base
+
+  attr_reader :import_definition
+
+
+
+  def initialize(import_definition)
+    @import_definition = import_definition
+  end
+
+
+
+  def import
+    external_table = Cranium::ExternalTable.new Cranium.application.sources[import_definition.name], Cranium::Database.connection
+
+    external_table.create
+    number_of_items_imported = import_from external_table.name
+    external_table.destroy
+
+    number_of_items_imported
+  end
+
+
+
+  protected
+
+  def import_from(external_table)
+    raise StandardError "Not implemented"
+  end
+
+
+
+  def database
+    Cranium::Database.connection
+  end
+
+
+
+  def target_table
+    import_definition.into
+  end
+
+
+
+  def source_fields
+    import_definition.field_associations.keys.map &:to_sym
+  end
+
+
+
+  def target_fields
+    import_definition.field_associations.values.map &:to_sym
+  end
+
+end

data/lib/cranium/import_strategy/delete_insert.rb

@@ -0,0 +1,40 @@
+class Cranium::ImportStrategy::DeleteInsert < Cranium::ImportStrategy::Base
+
+  def import_from(source_table)
+    @source_table = source_table
+
+    delete_existing_records
+    import_new_records
+    database[@source_table].count
+  end
+
+
+
+  private
+
+  def delete_existing_records
+    database.
+        from(Sequel.as(target_table, "target"), Sequel.as(@source_table, "source")).
+        where(delete_by_fields.qualify keys_with: :source, values_with: :target).
+        delete
+  end
+
+
+
+  def import_new_records
+    database.run database[target_table].insert_sql(target_fields, database[@source_table].select(*source_fields))
+  end
+
+
+
+  def delete_by_fields
+    Cranium::Sequel::Hash[delete_field_mapping]
+  end
+
+
+
+  def delete_field_mapping
+    import_definition.field_associations.select { |_, target_field| import_definition.delete_insert_on.include? target_field }
+  end
+
+end

data/lib/cranium/import_strategy/delta.rb

@@ -0,0 +1,8 @@
+class Cranium::ImportStrategy::Delta < Cranium::ImportStrategy::Base
+
+  def import_from(source_table)
+    database.run database[import_definition.into].insert_sql(target_fields, database[source_table].select(*source_fields))
+    database[source_table].count
+  end
+
+end

data/lib/cranium/import_strategy/merge.rb

@@ -0,0 +1,50 @@
+class Cranium::ImportStrategy::Merge < Cranium::ImportStrategy::Base
+
+  def import_from(source_table)
+    @source_table = source_table
+
+    update_existing_records
+    import_new_records
+    database[@source_table].count
+  end
+
+
+
+  private
+
+  def update_existing_records
+    database.
+      from(Sequel.as(target_table, "target"), Sequel.as(@source_table, "source")).
+      where(merge_fields.qualify keys_with: :source, values_with: :target).
+      update(not_merge_fields.qualify(keys_with: :source).invert)
+  end
+
+
+
+  def import_new_records
+    database.run database[target_table].insert_sql(target_fields,
+                                                   database[@source_table].
+                                                     left_outer_join(target_table, merge_fields.invert).
+                                                     where(merge_fields_are_empty).
+                                                     select(*source_fields).qualify)
+  end
+
+
+
+  def merge_fields
+    Cranium::Sequel::Hash[import_definition.merge_fields]
+  end
+
+
+
+  def not_merge_fields
+    Cranium::Sequel::Hash[import_definition.field_associations.reject { |key, _| merge_fields.keys.include? key }]
+  end
+
+
+
+  def merge_fields_are_empty
+    Hash[merge_fields.qualified_values(target_table).zip Array.new(merge_fields.count, nil)]
+  end
+
+end