blue-shift 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6eccf1e183a70d001c5616e75ba3f9b58a32b8d2
-  data.tar.gz: 76375171326b20569bb81538873d4d33f17ab306
+  metadata.gz: 7a52af4406d213a8437391b173949c1a052275e6
+  data.tar.gz: c9626f7ca6049730d8afbabe686b515b9e4a9ecf
 SHA512:
-  metadata.gz: 968c3d19b343b6069a0d9f6f6def75770dc35bb3ad4aa1f6fafe667b33b2ddde8c3d60ea8389ebec0e0a5ceb9459f19771268f27b70906260b7498854fcd595c
-  data.tar.gz: abca983bf558a8e70fd4ff2e2bd31f98ed11da6c4fdc5c8f414e136a915db27396ccd0a2f9f7ac019bc5e1c634d4da644fa459eacb393ba4e9d3cb99eb133620
+  metadata.gz: 75d418562c8142619bb5e055d1645e75c8c5ca362f1dfbd2601339ca4c50103768f81a8afe28ce8920ac8b0ccc95fb18121810a47ad6f7974ff07f4a1843de2c
+  data.tar.gz: d96f4afb5b6cc5963eaa3c8693fe6d28f714724e5a435bb8affdd148211fb4dd953a4568cbf58be09eb3a3f3d23d46baeb4f213bd8d54b2399c5330106abc662

data/.gitignore

@@ -7,3 +7,5 @@
 /pkg/
 /spec/reports/
 /tmp/
+.env
+*.log

data/Gemfile

@@ -3,3 +3,9 @@ source 'https://rubygems.org'
 gem 'guard-rspec', group: :development
 # Specify your gem's dependencies in blueshift.gemspec
 gemspec
+
+group :test do
+  gem 'pry'
+end
+
+gem 'dotenv'

data/README.md

@@ -26,9 +26,9 @@ Or install it yourself as:
 
     The Distribution Key. When specified, the :diststyle is set to :key unless otherwise specified
 
-- `:diststyle` => `:even` (default), `:key`, or `:all`
+- `:diststyle` => `:even` (implicit default), `:key`, or `:all`
 
-    The Distribution Style. This option has no effect unless `:distkey` is also specified
+    The Distribution Style. When `:distkey` is also specified, only `:key` DISTSTYLE is supported by Redshift.
 
 - `:sortkeys` => a list of column names
 
@@ -54,45 +54,32 @@ end
 
 ### Migrations (coming soon)
 
-Blueshift unifies migrations for your Postgres and Redshift databases into one file. Postgres migrations use ActiveRecord::Migration and Redshift uses Sequel.
-(I hate that this is separated, but that is the state that our app currently exists in until we sort it out.)
+Blueshift unifies migrations for your Postgres and Redshift databases into one file. Postgres migrations and Redshift migrations use Sequel.
 
 ```ruby
 Blueshift.migration do
   up do
-    # applies to Postgres + Redshift
-  end
-
-  down do
-    # applies to Postgres + Redshift
-  end
-end
-```
-
-If you want different migration behaviours for Redshift than for Postgres, you can override them by using `redup` and `reddown`:
-
-```ruby
-Blueshift.migration do
-  up do
-    # applies to Postgres only, because redup is defined below
+    # applies to Postgres only
   end
 
   down do
-    # applies to Postgres + Redshift
+    # applies to Postgres only
   end
   
   redup do
     # applies to Redshift only
   end
+  
+  reddown do
+    # applies to Redshift only
+  end
 end
 ```
 
-If you want your migration to only run on Postgres, you can specify it as an argument for the migration block:
+If you want your migration to only run on Postgres, you can specify an empty block:
 
 ```ruby
 Blueshift.migration do
-  pg_only!
-  
   up do
     # applies to Postgres only
   end
@@ -100,6 +87,10 @@ Blueshift.migration do
   down do
     # applies to Postgres only
   end
+  
+  redup {}
+  
+  reddown {}
 end
 ```
        

data/Rakefile

@@ -1,5 +1,6 @@
 require "bundler/gem_tasks"
 require "rspec/core/rake_task"
+Dir['lib/tasks/*.rake'].each{ |f| import f }
 
 RSpec::Core::RakeTask.new(:spec)
 

data/blueshift.gemspec

@@ -18,10 +18,11 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
-  spec.add_dependency 'sequel', '~> 0'
+  spec.add_dependency 'sequel', '~> 4'
   spec.add_dependency 'pg', '~> 0'
 
   spec.add_development_dependency 'bundler', '~> 1.10'
   spec.add_development_dependency 'rake', '~> 10.0'
   spec.add_development_dependency 'rspec', '~> 3.0'
+  spec.add_development_dependency 'dotenv', '~> 2.1.0'
 end

data/lib/blueshift.rb

@@ -1,4 +1,10 @@
 require 'blueshift/version'
+require 'blueshift/migration'
 require 'sequel'
 require 'sequel/adapters/redshift'
 
+module Blueshift
+  def self.migration(&block)
+    Blueshift::Migration.new(&block)
+  end
+end

data/lib/blueshift/migration.rb

@@ -0,0 +1,59 @@
+require 'sequel'
+require 'sequel/extensions/migration'
+require 'logger'
+
+module Blueshift
+  REDSHIFT_DB = Sequel.connect(ENV.fetch('REDSHIFT_URL'), logger: Logger.new('redshift.log'))
+  POSTGRES_DB = Sequel.connect(ENV.fetch('DATABASE_URL'), logger: Logger.new('postgres.log'))
+
+  class Migration
+    attr_reader :postgres_migration, :redshift_migration
+    MIGRATION_DIR = File.join(Dir.pwd, 'db/migrations')
+
+    def initialize(&block)
+      @postgres_migration = Sequel::SimpleMigration.new
+      @redshift_migration = Sequel::SimpleMigration.new
+
+      Sequel::Migration.descendants << self
+      instance_eval(&block)
+      validate!
+    end
+
+    def up(&block)
+      postgres_migration.up = block
+    end
+
+    def down(&block)
+      postgres_migration.down = block
+    end
+
+    def redup(&block)
+      redshift_migration.up = block
+    end
+
+    def reddown(&block)
+      redshift_migration.down = block
+    end
+
+    def apply(db, direction)
+      if db.is_a?(Sequel::Redshift::Database)
+        redshift_migration.apply(db, direction)
+      else
+        postgres_migration.apply(db, direction)
+      end
+    end
+
+    def self.run!
+      Sequel::Migrator.run(POSTGRES_DB, MIGRATION_DIR, column: :postgres_version)
+      Sequel::Migrator.run(REDSHIFT_DB, MIGRATION_DIR, column: :redshift_version)
+    end
+
+    private
+
+    def validate!
+      unless [postgres_migration.up, postgres_migration.down, redshift_migration.up, redshift_migration.down].all?
+        raise ArgumentError, 'must declare blocks for up, down, redup, and reddown'
+      end
+    end
+  end
+end

data/lib/blueshift/version.rb

@@ -1,3 +1,3 @@
 module Blueshift
-  VERSION = '0.0.1'
+  VERSION = '0.0.2'
 end

data/lib/sequel/adapters/redshift.rb

@@ -9,12 +9,6 @@ module Sequel
       SORTSTYLES = [:compound, :interleaved].freeze
       DISTSTYLES = [:even, :key, :all].freeze
 
-      def column_definition_primary_key_sql(sql, column)
-        result = super
-        result << ' IDENTITY' if result
-        result
-      end
-
       def serial_primary_key_options
         # redshift doesn't support serial type
         super.merge(serial: false)
@@ -29,15 +23,21 @@ module Sequel
       def create_table_sql(name, generator, options)
         validate_options!(options)
         super.tap do |sql|
+          sql << diststyle_sql(options)
           sql << distkey_sql(options)
           sql << sortstyle_sql(options)
         end
       end
 
+      def diststyle_sql(options)
+        if options[:diststyle]
+          " DISTSTYLE #{options[:diststyle].to_s.upcase}"
+        end.to_s
+      end
+
       def distkey_sql(options)
         if options[:distkey]
-          options[:diststyle] ||= :key
-          " DISTSTYLE #{options[:diststyle].to_s.upcase} DISTKEY (#{options[:distkey]})"
+          " DISTKEY (#{options[:distkey]})"
         end.to_s
       end
 
@@ -56,6 +56,51 @@ module Sequel
       def invalid?(value, allowed)
         value && !allowed.include?(value)
       end
+
+      # OVERRIDE for Redshift. Now always expect the "id" column to be the primary key
+      # The dataset used for parsing table schemas, using the pg_* system catalogs.
+      def schema_parse_table(table_name, opts)
+        m = output_identifier_meth(opts[:dataset])
+        ds = metadata_dataset.select(:pg_attribute__attname___name,
+                                     SQL::Cast.new(:pg_attribute__atttypid, :integer).as(:oid),
+                                     SQL::Cast.new(:basetype__oid, :integer).as(:base_oid),
+                                     SQL::Function.new(:format_type, :basetype__oid, :pg_type__typtypmod).as(:db_base_type),
+                                     SQL::Function.new(:format_type, :pg_type__oid, :pg_attribute__atttypmod).as(:db_type),
+                                     SQL::Function.new(:pg_get_expr, :pg_attrdef__adbin, :pg_class__oid).as(:default),
+                                     SQL::BooleanExpression.new(:NOT, :pg_attribute__attnotnull).as(:allow_null),
+                                     SQL::Function.new(:COALESCE, SQL::BooleanExpression.from_value_pairs(:name => 'id'), false).as(:primary_key)).
+            from(:pg_class).
+            join(:pg_attribute, :attrelid=>:oid).
+            join(:pg_type, :oid=>:atttypid).
+            left_outer_join(:pg_type___basetype, :oid=>:typbasetype).
+            left_outer_join(:pg_attrdef, :adrelid=>:pg_class__oid, :adnum=>:pg_attribute__attnum).
+            left_outer_join(:pg_index, :indrelid=>:pg_class__oid, :indisprimary=>true).
+            filter(:pg_attribute__attisdropped=>false).
+            filter{|o| o.pg_attribute__attnum > 0}.
+            filter(:pg_class__oid=>regclass_oid(table_name, opts)).
+            order(:pg_attribute__attnum)
+        ds.map do |row|
+          row[:default] = nil if blank_object?(row[:default])
+          if row[:base_oid]
+            row[:domain_oid] = row[:oid]
+            row[:oid] = row.delete(:base_oid)
+            row[:db_domain_type] = row[:db_type]
+            row[:db_type] = row.delete(:db_base_type)
+          else
+            row.delete(:base_oid)
+            row.delete(:db_base_type)
+          end
+          row[:type] = schema_column_type(row[:db_type])
+          if row[:primary_key]
+            row[:auto_increment] = !!(row[:default] =~ /\Anextval/io)
+          end
+          [m.call(row.delete(:name)), row]
+        end
+      end
+
+      # OVERRIDE since indexes are not enforced by Redshift
+      def indexes(table, opts=OPTS)
+      end
     end
 
     class Dataset < Postgres::Dataset

data/lib/sequel/extensions/redshift_schema_dumper.rb

@@ -0,0 +1,52 @@
+Sequel.extension :schema_dumper
+
+module Sequel
+  module Redshift
+    module SchemaDumper
+      include Sequel::SchemaDumper
+
+      def dump_table_schema(table, options=OPTS)
+        gen = dump_table_generator(table, options)
+        commands = [gen.dump_columns, gen.dump_constraints, gen.dump_indexes].reject { |x| x == '' }.join("\n\n")
+
+        "create_table(#{table.inspect}#{table_options(table, gen, options)}) do\n#{commands.gsub(/^/o, '  ')}\nend"
+      end
+
+      def table_options(table, gen, options)
+        s = {distkey: table_distkey(table),
+         sortkeys: table_sortkeys(table),
+         sortstyle: table_sortstyle(table),
+         ignore_index_errors: (!options[:same_db] && options[:indexes] != false && !gen.indexes.empty?)
+        }.select { |_,v| v }.inspect[1...-1]
+
+        s.empty? ? s : ", #{s}"
+      end
+
+      private
+
+      def table_distkey(table)
+        key = pg_table_def(table).filter(distkey: true).map(:column).first
+        key.to_sym if key
+      end
+
+      def table_sortkeys(table)
+        keys = sortkey_columns(table).map{ |r| r[:column].to_sym }
+        keys unless keys.empty?
+      end
+
+      def table_sortstyle(table)
+        :interleaved if sortkey_columns(table).any? { |row| row[:sortkey] < 0 }
+      end
+
+      def sortkey_columns(table)
+        pg_table_def(table).exclude(sortkey: 0).order(Sequel.function(:abs, :sortkey))
+      end
+
+      def pg_table_def(table)
+        self[:pg_table_def].where(schemaname: 'public', tablename: table.to_s).select(:column, :sortkey)
+      end
+    end
+  end
+
+  Database.register_extension(:redshift_schema_dumper, Redshift::SchemaDumper)
+end

data/lib/tasks/generate_migration.rake

@@ -0,0 +1,22 @@
+require 'fileutils'
+
+namespace :blueshift do
+  namespace :g do
+    desc 'Generate a timestamped, empty Blueshift migration.'
+    task :migration, :name do |_, args|
+      if args[:name].nil?
+        puts 'You must specify a migration name (e.g. rake generate:migration[create_events])!'
+        exit false
+      end
+
+      content = "Blueshift.migration do\n  up do\n    \n  end\n\n  down do\n    \n  end\n\n  redup do\n    \n  end\n\n  reddown do\n    \n  end\nend\n"
+      timestamp = Time.now.strftime('%Y%m%d%H%M%S')
+      filename = File.join(Dir.pwd, 'db/migrations', "#{timestamp}_#{args[:name]}.rb")
+
+      FileUtils.mkdir_p(File.dirname(filename))
+      File.open(filename, 'w') { |f| f << content }
+
+      puts "Created the migration #{filename}"
+    end
+  end
+end

data/lib/tasks/schema.rake

@@ -0,0 +1,50 @@
+require 'dotenv'
+Dotenv.load
+require 'fileutils'
+require 'blueshift'
+
+path = File.join(Dir.pwd, 'db')
+
+task :ensure_db_dir do
+  FileUtils.mkdir_p(path)
+end
+
+namespace :pg do
+  namespace :schema do
+    desc 'Dumps the Postgres schema to a file'
+    task :dump => :ensure_db_dir do
+      Blueshift::POSTGRES_DB.extension :redshift_schema_dumper
+      File.open(File.join(path, 'schema.rb'), 'w') { |f| f << Blueshift::POSTGRES_DB.dump_schema_migration(same_db: true) }
+    end
+
+    desc 'Loads the Postgres schema from the file to the database'
+    task :load => :ensure_db_dir do
+      eval(File.join(path, 'schema.rb')).apply(Blueshift::POSTGRES_DB, :up)
+    end
+  end
+end
+
+
+namespace :redshift do
+  namespace :schema do
+    desc 'Dumps the Postgres schema to a file'
+    task :dump => :ensure_db_dir do
+      Blueshift::REDSHIFT_DB.extension :redshift_schema_dumper
+      File.open(File.join(path, 'schema_redshift.rb'), 'w') { |f| f << Blueshift::REDSHIFT_DB.dump_schema_migration(same_db: true) }
+    end
+
+    desc 'Loads the Postgres schema from the file to the database'
+    task :load => :ensure_db_dir do
+      eval(File.join(path, 'schema_redshift.rb')).apply(Blueshift::REDSHIFT_DB, :up)
+    end
+  end
+end
+
+namespace :blueshift do
+  desc 'Runs migrations for both Postgres and Redshift'
+  task :migrate do
+    puts 'Running migrations for Postgres and Redshift...', ''
+    Blueshift::Migration.run!
+  end
+end
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: blue-shift
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Gabriel Mansour
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-03-07 00:00:00.000000000 Z
+date: 2016-03-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sequel
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '4'
 - !ruby/object:Gem::Dependency
   name: pg
   requirement: !ruby/object:Gem::Requirement
@@ -80,6 +80,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: dotenv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.1.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.1.0
 description: Amazon Redshift adapter for Sequel
 email:
 - dev+gabriel@influitive.com
@@ -101,8 +115,12 @@ files:
 - bin/setup
 - blueshift.gemspec
 - lib/blueshift.rb
+- lib/blueshift/migration.rb
 - lib/blueshift/version.rb
 - lib/sequel/adapters/redshift.rb
+- lib/sequel/extensions/redshift_schema_dumper.rb
+- lib/tasks/generate_migration.rake
+- lib/tasks/schema.rake
 homepage: https://github.com/influitive/blueshift
 licenses: []
 metadata: {}