bricolage 5.17.0 → 5.17.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 05fc1225db07602f098fbcd2271156076265a888
-  data.tar.gz: 11e34d16227dfcdb02bd1d51043d99897ca0f133
+  metadata.gz: 4032da3e6706292157209142c6b52553b460773c
+  data.tar.gz: 8b629e29d61199dddb5575e5dc642f686b237fda
 SHA512:
-  metadata.gz: 516296b703b34e4dc187f78d38d6087f01ae69867717677530ef49cf7e31e8a38ece329fc273af73891aec30d76887b1b61eda977fb26985ecb4d242aceb3fd1
-  data.tar.gz: af04b494591374c139e1a66fce78704722ff522ee3b948c72039048d4decbd283d85710ab9da203d4a89ca4dac5b212aadb7b62f3bb986539f37cb616c3d9d78
+  metadata.gz: e6cb089f2d6b6f89034e17bcd26097265a75cc169e892f36b9f76cf4518da6ee5d8d584364f092fe0115781aad57157ccb98d0def50ed7a39f9a8c78e133d6d2
+  data.tar.gz: 4b6d42945c0151269e7e9c0f3c9f875587900db905dc54cdcab6b25bd0e7581cebdc00f601bcbc2ff3176a09266c3f33718a287e5578d55f2adb9db97ff274a0

data/lib/bricolage/commandlineapplication.rb

@@ -0,0 +1,80 @@
+require 'bricolage/context'
+require 'forwardable'
+
+module Bricolage
+  class CommandLineApplication
+    def CommandLineApplication.define
+      opts = new
+      yield opts
+      opts.parse!
+      opts
+    end
+
+    def initialize
+      @name = File.basename($0)
+      @home = nil
+      @env = nil
+      @subsys = nil
+      @ds = {}
+      @options = OptionParser.new
+      @options.banner = "Usage: #{@name} [options]"
+      define_default_options
+    end
+
+    attr_reader :name
+
+    def define_default_options
+      @options.on_tail('-e', '--environment=ENV', "Bricolage execution environment. (default: #{Context.environment})") {|env|
+        @env = env
+      }
+      default_home = Context.home_path
+      @options.on_tail('-C', '--home=PATH', "Bricolage home directory. (default: #{default_home == Dir.getwd ? '.' : default_home})") {|path|
+        @home = path
+      }
+      @options.on_tail('--help', 'Prints this message and quit.') {
+        puts @options.help
+        exit 0
+      }
+    end
+
+    extend Forwardable
+    def_delegator '@options', :on
+
+    DataSourceOpt = Struct.new(:kind, :name, :ds)
+
+    def data_source_option(long_option, description, kind:, short: nil, default: nil)
+      @ds[long_option] = DataSourceOpt.new(kind, default || kind)
+      @options.on(* [short, long_option + "=NAME", description + " (default: #{default || kind})"].compact) {|ds_name|
+        @ds[long_option] = DataSourceOpt.new(kind, ds_name)
+      }
+    end
+
+    def data_source(long_option)
+      ent = @ds[long_option] or raise ArgumentError, "no such data source entry: #{long_option}"
+      ent.ds ||= context.get_data_source(ent.kind, ent.name)
+    end
+
+    def parse!
+      return @options.parse!
+    rescue OptionParser::ParseError => err
+      $stderr.puts "#{$0}: error: #{err.message}"
+      $stderr.puts @options.help
+      exit 1
+    end
+
+    def context
+      @context ||= create_context
+    end
+
+    def create_context
+      Bricolage::Context.for_application(@home, @env)
+    end
+
+    def main
+      yield
+    rescue => ex
+      $stderr.puts "#{@name}: error: #{ex.message}"
+      exit 1
+    end
+  end
+end

data/lib/bricolage/context.rb

@@ -3,6 +3,7 @@ require 'bricolage/datasource'
 require 'bricolage/variables'
 require 'bricolage/configloader'
 require 'bricolage/logger'
+require 'bricolage/exception'
 require 'forwardable'
 
 module Bricolage
@@ -10,13 +11,24 @@ module Bricolage
   class Context
     DEFAULT_ENV = 'development'
 
-    def Context.environment(opt_env)
+    def Context.environment(opt_env = nil)
       opt_env || ENV['BRICOLAGE_ENV'] || DEFAULT_ENV
     end
 
-    def Context.for_application(home_path, job_path = nil, environment: nil, global_variables: nil, logger: nil)
+    def Context.home_path(opt_path = nil)
+      FileSystem.home_path(opt_path)
+    end
+
+    def Context.for_application(home_path = nil, job_path_0 = nil, job_path: nil, environment: nil, global_variables: nil, logger: nil)
       env = environment(environment)
-      fs = FileSystem.for_option_pathes(home_path, job_path, env)
+      if (job_path ||= job_path_0)
+        fs = FileSystem.for_job_path(job_path, env)
+        if home_path and home_path.realpath.to_s != fs.home_path.realpath.to_s
+          raise OptionError, "--home option and job file is exclusive"
+        end
+      else
+        fs = FileSystem.for_options(home_path, env)
+      end
       load(fs, env, global_variables: global_variables, logger: logger)
     end
 

data/lib/bricolage/filesystem.rb

@@ -5,24 +5,21 @@ require 'pathname'
 module Bricolage
 
   class FileSystem
-    def FileSystem.for_option_pathes(home_path, job_path, env)
-      if job_path
-        home, subsys_id = extract_home_dirs(job_path)
-        if home_path and home_path.realpath.to_s != home.realpath.to_s
-          raise OptionError, "--home option and job file is exclusive"
-        end
-        new(home, env).subsystem(subsys_id)
-      elsif home_path
-        new(home_path, env)
-      elsif home = ENV['BRICOLAGE_HOME']
-        new(home, env)
-      else
-        new(Pathname.getwd, env)
-      end
+    def FileSystem.home_path(opt_path = nil)
+      Pathname(opt_path || ENV['BRICOLAGE_HOME'] || Dir.getwd)
+    end
+
+    def FileSystem.for_options(home, env)
+      new(home_path(home), env)
+    end
+
+    def FileSystem.for_job_path(job_path, env)
+      home, subsys_id = parse_job_path(job_path)
+      new(home, env).subsystem(subsys_id)
     end
 
     # job_path -> [home_path, subsys_id]
-    def FileSystem.extract_home_dirs(job_path)
+    def FileSystem.parse_job_path(job_path)
       subsys_path = Pathname(job_path).realpath.parent
       return subsys_path.parent, subsys_path.basename
     rescue SystemCallError => err

data/lib/bricolage/mysqldatasource.rb

@@ -219,6 +219,11 @@ module Bricolage
         @statement.bind(*args) if @statement
       end
 
+      def source
+        "-- myexport #{@table} -> #{@s3ds.bucket_name}/#{@prefix}" +
+          (@statement ? "\n#{@statement.stripped_source}" : "")
+      end
+
       def s3export
         cmd = build_cmd(command_parameters)
         ds.logger.info '[CMD] ' + cmd.join(' ')

data/lib/bricolage/s3datasource.rb

@@ -1,5 +1,6 @@
 require 'bricolage/datasource'
 require 'bricolage/commandutils'
+require 'bricolage/exception'
 require 'aws-sdk'
 require 'stringio'
 
@@ -29,8 +30,8 @@ module Bricolage
       @iam_role = iam_role
       @master_symmetric_key = master_symmetric_key
       @encryption = encryption
-      @s3cfg = s3cfg
-      @configurations = @s3cfg ? load_configurations(@s3cfg) : nil
+      @s3cfg_path = s3cfg
+      @s3cfg = @s3cfg_path ? load_s3cfg(@s3cfg_path) : nil
     end
 
     attr_reader :endpoint
@@ -42,32 +43,42 @@ module Bricolage
       S3Task.new(self)
     end
 
-    # For Redshift
+    # For Redshift COPY/UNLOAD
     def credential_string
       if @iam_role
         "aws_iam_role=#{@iam_role}"
-      else
+      elsif access_key
         [
           "aws_access_key_id=#{access_key}",
           "aws_secret_access_key=#{secret_key}",
           (@master_symmetric_key && "master_symmetric_key=#{@master_symmetric_key}")
         ].compact.join(';')
+      else
+        raise ParameterError, "[s3:#{@bucket_name}] credential string requested but no credentials exist"
       end
     end
 
+    # AWS access key ID.
+    # This property may be nil, we can use EC2 instance or ECS task attached IAM role in that case.
     def access_key
-      @access_key_id || get_config('access_key')
+      @access_key_id || get_s3cfg('access_key')
     end
 
+    # AWS secret access key.
     def secret_key
-      @secret_access_key || get_config('secret_key')
+      @secret_access_key || get_s3cfg('secret_key')
     end
 
-    def get_config(key)
-      @configurations[key] or raise ParameterError, "missing s3cfg entry: #{key}"
+    # Redshift attached IAM role ARN
+    attr_reader :iam_role
+
+    def get_s3cfg(key)
+      return nil unless @s3cfg
+      @s3cfg[key] or raise ParameterError, "[s3:#{@bucket_name}] missing s3cfg entry: #{key}"
     end
+    private :get_s3cfg
 
-    def load_configurations(path)
+    def load_s3cfg(path)
       h = {}
       File.foreach(path) do |line|
         case line
@@ -79,6 +90,7 @@ module Bricolage
       end
       h
     end
+    private :load_s3cfg
 
     attr_reader :encryption
 

data/lib/bricolage/version.rb

@@ -1,4 +1,4 @@
 module Bricolage
   APPLICATION_NAME = 'Bricolage'
-  VERSION = '5.17.0'
+  VERSION = '5.17.1'
 end

data/test/home/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ../..
   specs:
-    bricolage (5.17.0)
+    bricolage (5.17.1)
       aws-sdk (~> 2)
       mysql2
       pg

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bricolage
 version: !ruby/object:Gem::Version
-  version: 5.17.0
+  version: 5.17.1
 platform: ruby
 authors:
 - Minero Aoki
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-09-14 00:00:00.000000000 Z
+date: 2016-09-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg
@@ -155,6 +155,7 @@ files:
 - jobclass/wait-file.rb
 - lib/bricolage.rb
 - lib/bricolage/application.rb
+- lib/bricolage/commandlineapplication.rb
 - lib/bricolage/commandutils.rb
 - lib/bricolage/configloader.rb
 - lib/bricolage/context.rb