redshift_etl 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: dba6f0ba7cea3305daeaa95a294202898e7629b0
+  data.tar.gz: 65557d8fbdab5bc5f5b4e71da8076204ecc67b94
+SHA512:
+  metadata.gz: 336794a6b640282754b8aab124819c2593df3d5e00f4a210bc9cdbec5e1ff910ffa7f60d75fb8fe9461d152b272d80980d2c23b0989c292b06d78bffcee82b46
+  data.tar.gz: 8021e32c16bc1dc3f3a4e161b89171e5261a84e84b1191ce9e2a9be013652ea1b466597b34e5eb5fa795bbf43c3ebfaa2e3c958d56a226307c13a951acbd1751

data/Gemfile

@@ -0,0 +1,2 @@
+source 'http://rubygems.org'
+gemspec

data/Gemfile.lock

@@ -0,0 +1,42 @@
+PATH
+  remote: .
+  specs:
+    redshift_etl (0.0.1)
+      activesupport
+      aws-sdk (~> 1)
+      pg
+      que (= 0.9.0)
+      sequel
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    activesupport (4.2.3)
+      i18n (~> 0.7)
+      json (~> 1.7, >= 1.7.7)
+      minitest (~> 5.1)
+      thread_safe (~> 0.3, >= 0.3.4)
+      tzinfo (~> 1.1)
+    aws-sdk (1.64.0)
+      aws-sdk-v1 (= 1.64.0)
+    aws-sdk-v1 (1.64.0)
+      json (~> 1.4)
+      nokogiri (>= 1.4.4)
+    i18n (0.7.0)
+    json (1.8.3)
+    mini_portile (0.6.2)
+    minitest (5.8.0)
+    nokogiri (1.6.6.2)
+      mini_portile (~> 0.6.0)
+    pg (0.18.2)
+    que (0.9.0)
+    sequel (4.25.0)
+    thread_safe (0.3.5)
+    tzinfo (1.2.2)
+      thread_safe (~> 0.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  redshift_etl!

data/TODO

@@ -0,0 +1,29 @@
+- Use Sequel instead of shell+psql+sql to dump / load data
+- Use ruby gem for s3 uploads
+- Use proper temp directory / file handling
+- TMPDIR should be configurable via envvar if that's not automatic
+- Fail on errors, don't silently fail and continue (probably handled by using gems instead of shell commands)
+- Support incremental uploads
+- Add way to automatically run (passing in the time period we want to process updates for)
+- Use a temporary s3 folder
+   - when upload completes, move to a backup place
+   - on finish, delete temporary folder (and files)
+
++ Add way to automatically generate the config files based off schema
++ More tables!
++ Do a load to a temp table, then replace table with new one (or whatever redshift docs say to do)
+
+
+sql generator changes:
+  - switch to sequel
+  - trim text length to 64k
+
+incremental:
+  - specify the column to do the timestamp on
+  - specify the primary key to do the replacement on
+  - pass this information into the yaml generator
+  - extract changes:
+     * change the filename to include the day (today at 12 am to current time)
+     * query should select on the timestamp
+  - load changes:
+    * instead of drop/create,rename, intead do a 'delete from where primary key=..; insert into..'

data/go.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+for file in config/*/*.yml
+do
+  envdir /etc/env/redshift bundle exec /usr/local/bin/ruby bin/redshift_etl.rb $file
+done

data/lib/redshift_etl.rb

@@ -0,0 +1,8 @@
+require_relative './redshift_etl/config'
+require_relative './redshift_etl/extract'
+require_relative './redshift_etl/transform'
+require_relative './redshift_etl/load'
+require_relative './redshift_etl/archive'
+
+module RedshiftETL
+end

data/lib/redshift_etl/archive.rb

@@ -0,0 +1,15 @@
+module RedshiftETL
+  module Archive
+    def self.call(config)
+      # Delete previously-stored files if doing a full update.
+      config.s3_bucket.objects.with_prefix(config.final_s3_path).delete_all
+
+      # Move objects from temp dir to final location.
+      config.s3_bucket.objects.with_prefix(config.temp_s3_path).each do |object|
+        filename = File.basename(object.key)
+        new_location = config.final_s3_path + filename
+        object.move_to(new_location)
+      end
+    end
+  end
+end

data/lib/redshift_etl/config.rb

@@ -0,0 +1,114 @@
+require 'yaml'
+require 'active_support'
+require 'active_support/core_ext'
+require 'securerandom'
+require 'time'
+require 'aws-sdk'
+
+module RedshiftETL
+  class Config
+    def initialize(file_path)
+      fail "plz provide a file" if file_path.blank?
+      @config = YAML.load(File.open(file_path))
+    rescue
+      puts $!
+      fail "Did you provide a valid YAML file?"
+    end
+
+    def temp_s3_path
+      "tmp/#{redshift_schema}/#{redshift_table}/#{temp_key}/"
+    end
+
+    def final_s3_path
+      "files/#{redshift_schema}/#{redshift_table}/"
+    end
+
+    def enabled?
+      @config["enabled"]
+    end
+
+    def disabled?
+      !enabled?
+    end
+
+    def primary_key
+      @config["primary_key"]
+    end
+
+    def incremental_updates?
+      @config["incremental_updates"]
+    end
+
+    def update_timestamp_column
+      @config["update_timestamp_column"]
+    end
+
+    def query
+      @config["query"]
+    end
+
+    def table_name
+      @config["table_name"]
+    end
+
+    def connection_string
+      @config["connection_string"]
+    end
+
+    def filename
+      @config["filename"] + '.csv'
+    end
+
+    def redshift_table
+      @config["redshift_table"]
+    end
+
+    def redshift_schema
+      @config["redshift_schema"]
+    end
+
+    def columns
+      @config["columns"]
+    end
+
+    def redshift_connection
+      ENV['REDSHIFT_DATABASE_URL']
+    end
+
+    def s3_access_key_id
+      ENV['AWS_ACCESS_KEY_ID']
+    end
+
+    def s3_secret_access_key
+      ENV['AWS_SECRET_ACCESS_KEY']
+    end
+
+    def s3_region
+      "us-east-1"
+    end
+
+    def s3_bucket_name
+      "tanga-redshift"
+    end
+
+    def check!
+      fail "plz set AWS_SECRET_ACCESS_KEY" if s3_secret_access_key.blank?
+      fail "plz set REDSHIFT_DATABASE_URL" if redshift_connection.blank?
+      fail "plz set AWS_ACCESS_KEY_ID"     if s3_access_key_id.blank?
+    end
+
+    def s3
+      AWS::S3.new(access_key_id: s3_access_key_id, secret_access_key: s3_secret_access_key)
+    end
+
+    def s3_bucket
+      s3.buckets[s3_bucket_name]
+    end
+
+    private
+
+    def temp_key
+      @temp_key ||= Time.now.iso8601 + SecureRandom.hex[0..7]
+    end
+  end
+end

data/lib/redshift_etl/extract.rb

@@ -0,0 +1,15 @@
+require_relative './config'
+require_relative './extract/database_fetcher'
+require_relative './extract/file_maker'
+
+# Copies stuff from the database into a CSV file.
+module RedshiftETL
+  module Extract
+    def self.call(config)
+      db         = DatabaseFetcher.new(config)
+      file_maker = FileMaker.new(config)
+
+      file_maker.run(db)
+    end
+  end
+end

data/lib/redshift_etl/extract/database_fetcher.rb

@@ -0,0 +1,43 @@
+require 'sequel'
+
+module RedshiftETL
+  module Extract
+    # Copies data from a database in CSV format.
+    class DatabaseFetcher
+      attr_reader :config
+      def initialize(config)
+        @config = config
+      end
+
+      def lines
+        sequel do |db|
+          db.copy_table(dataset_to_copy(db), format: :csv, options: 'header') do |line|
+            begin
+              yield line
+            rescue # copy_table rescues errors, hard to debug without rescuing them ourselves.
+              puts $!.inspect
+              puts $!.backtrace
+              raise
+            end
+          end
+        end
+      end
+
+      def sequel(&block)
+        Sequel.connect(config.connection_string) do |db|
+          db << "set time zone 'UTC'"
+          block.call(db)
+        end
+      end
+
+      def dataset_to_copy(db)
+        table = db[config.table_name.to_sym]
+        # TODO truncate to 65k for varchar columns
+        # TODO maybe later, if citext, convert to lower(column)
+        columns_to_copy = config.columns.keys.map(&:to_sym)
+
+        table.select(*columns_to_copy)
+      end
+    end
+  end
+end

data/lib/redshift_etl/extract/file_maker.rb

@@ -0,0 +1,59 @@
+require 'zlib'
+require 'tempfile'
+
+module RedshiftETL
+  module Extract
+    # Extracts lines from the data source and puts the CSV files
+    # on S3.
+    class FileMaker
+      attr_reader :config
+      def initialize(config)
+        @config     = config
+        @file_index = 0
+        @line_index = 0
+      end
+
+      def run(data_source)
+        new_file!
+        data_source.lines do |line|
+          process_line(line)
+        end
+        process_file!
+      end
+
+      private
+
+      def process_line(line)
+        @line_index += 1
+
+        @header ||= line
+        @gzip.write(line)
+
+        if @line_index == 1_000_000
+          process_file!
+          new_file!
+          @gzip.write(@header)
+        end
+      end
+
+      def new_file!
+        @temp_file = Tempfile.new("#{config.table_name}.#{@file_index}.")
+        @gzip = Zlib::GzipWriter.open(@temp_file.path)
+        @file_index += 1
+        @line_index = 0
+      end
+
+      def process_file!
+        @gzip.close
+        upload_to_s3
+        @temp_file.close!
+        GC.start
+      end
+
+      def upload_to_s3
+        s3_path = config.temp_s3_path + "#{config.filename}.#{@file_index}.gz"
+        config.s3_bucket.objects[s3_path].write(File.open(@temp_file))
+      end
+    end
+  end
+end

data/lib/redshift_etl/load.rb

@@ -0,0 +1,72 @@
+#!/usr/bin/env ruby
+
+# Loads stuff from CSV files into Redshift
+module RedshiftETL
+  # TODO: break into some objects
+  module Load
+    def self.call(config)
+      load_into_redshift(config)
+    end
+
+    def self.load_into_redshift(config)
+      sql_statements = []
+      sql_statements << "create schema if not exists #{config.redshift_schema};"
+      sql_statements << "set search_path=#{config.redshift_schema};"
+      sql_statements << "drop table if exists #{config.redshift_table}_staging;"
+      sql_statements << create_table(config)
+
+      sql_statements << "
+      copy #{staging_table_name(config)} from 's3://#{config.s3_bucket_name}/#{config.temp_s3_path}'
+      credentials 'aws_access_key_id=#{config.s3_access_key_id};aws_secret_access_key=#{config.s3_secret_access_key}'
+      csv
+      gzip
+      ignoreheader 1
+      trimblanks
+      statupdate on
+      truncatecolumns
+      timeformat 'auto'; "
+
+      if config.incremental_updates?
+        sql_statements << "
+          begin;
+
+            #{create_table_statement(config, config.redshift_table)}
+
+            delete from #{config.redshift_table}
+            using #{staging_table_name(config)}
+            where #{config.redshift_table}.#{config.primary_key} = #{staging_table_name(config)}.#{config.primary_key};
+
+            insert into #{config.redshift_table}
+            select * from #{staging_table_name(config)};
+
+            drop table #{staging_table_name(config)};
+
+          commit;
+          "
+      else
+        sql_statements << "drop table if exists #{ config.redshift_table};"
+        sql_statements << "alter table #{staging_table_name(config)} rename to #{config.redshift_table};"
+      end
+
+      Sequel.connect(config.redshift_connection + "?force_standard_strings=f&client_min_messages=") do |db|
+        sql_statements.each do |statement|
+          print db[statement].get
+        end
+      end
+    end
+
+    def self.create_table(config)
+      create_table_statement(config, staging_table_name(config))
+    end
+
+    def self.create_table_statement(config, table_name)
+      sql = "create table if not exists #{table_name} ("
+      sql << config.columns.map { |column, type| "#{column} #{type}" }.join(', ')
+      sql << "); "
+    end
+
+    def self.staging_table_name(config)
+      config.redshift_table + "_staging"
+    end
+  end
+end

data/lib/redshift_etl/test.rb

@@ -0,0 +1,7 @@
+require_relative './config'
+require_relative './extract'
+require_relative './load'
+
+config = RedshiftETL::Config.new('config/tanga/base_skus.yml')
+puts RedshiftETL::Extract.call(config)
+puts RedshiftETL::Load.call(config)

data/lib/redshift_etl/transform.rb

@@ -0,0 +1,7 @@
+module RedshiftETL
+  module Transform
+    def self.call(_config)
+      # Empty for now.
+    end
+  end
+end

data/redshift.gemspec

@@ -0,0 +1,18 @@
+$:.push File.expand_path("../lib", __FILE__)
+
+# Describe your gem and declare its dependencies:
+Gem::Specification.new do |s|
+  s.name        = "redshift_etl"
+  s.version     = "0.0.1"
+  s.authors     = ["Joe Van Dyk"]
+  s.email       = ["joe@tanga.com"]
+  s.homepage    = "https://www.tanga.com"
+  s.summary     = "redshift etl"
+  s.require_paths = ["lib"]
+  s.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features|config|bin)/}) }
+  s.add_dependency 'que', '0.9.0'
+  s.add_dependency 'activesupport'
+  s.add_dependency 'sequel'
+  s.add_dependency 'pg'
+  s.add_dependency 'aws-sdk', '~> 1'
+end

metadata

@@ -0,0 +1,128 @@
+--- !ruby/object:Gem::Specification
+name: redshift_etl
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Joe Van Dyk
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-08-11 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: que
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.9.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.9.0
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sequel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+description: 
+email:
+- joe@tanga.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- Gemfile.lock
+- TODO
+- go.sh
+- lib/redshift_etl.rb
+- lib/redshift_etl/archive.rb
+- lib/redshift_etl/config.rb
+- lib/redshift_etl/extract.rb
+- lib/redshift_etl/extract/database_fetcher.rb
+- lib/redshift_etl/extract/file_maker.rb
+- lib/redshift_etl/load.rb
+- lib/redshift_etl/test.rb
+- lib/redshift_etl/transform.rb
+- redshift.gemspec
+homepage: https://www.tanga.com
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: redshift etl
+test_files: []
+has_rdoc: