bricolage 5.8.7

data/lib/bricolage/tddatasource.rb

@@ -0,0 +1,116 @@
+require 'bricolage/datasource'
+require 'bricolage/commandutils'
+require 'stringio'
+require 'date'
+
+module Bricolage
+
+  class TDDataSource < DataSource
+    declare_type 'td'
+
+    include CommandUtils
+
+    def initialize(database: nil, username: nil, apikey: nil, td: 'td', priority: -2)
+      @database = database
+      @apikey = apikey
+      @td = td
+      @priority = priority
+    end
+
+    def new_task
+      TDTask.new(self)
+    end
+
+    def query_command_args(*args)
+      [@td, "--apikey=#{@apikey}", "query", "--database=#{@database}", "--wait"] + args
+    end
+
+    def delete_command_args(*args)
+      [@td, "--apikey=#{@apikey}", "table:partial_delete", @database, *args, "--wait"]
+    end
+
+    def exec(*args)
+      JobResult.for_process_status(command(*args))
+    end
+  end
+
+  class TDTask < DataSourceTask
+    def export(stmt, path: nil, **opts)
+      add TDTask::Export.new(stmt, path: path, **opts)
+    end
+
+    class Export < Action
+      include CommandUtils
+
+      def initialize(stmt, path: nil, format: nil, gzip: false, override: false)
+        @statement = stmt
+        @path = path
+        @format = format
+        @gzip = gzip
+        @override = override
+      end
+
+      def_delegator '@statement', :bind
+
+      def source
+        buf = StringIO.new
+        buf.puts command_args(new_tmpfile_path).join(' ') + " <<EndTDSQL"
+        buf.puts @statement.source
+        buf.puts 'EndTDSQL'
+        buf.string
+      end
+
+      GZIP_COMMAND = 'gzip'   # FIXME: parameterize
+
+      def run
+        if File.exist?(@path) and not @override
+          raise JobFailure, "target file exists: #{@path.inspect}"
+        end
+        puts @statement.source
+        td_result = make_tmpfile(@statement.source) {|query_path|
+          ds.exec(*command_args(query_path))
+        }
+        if @gzip
+          gzip_result = ds.command(GZIP_COMMAND, export_path)
+          raise JobFailure, "gzip failed" unless gzip_result.success?
+        end
+        td_result
+      end
+
+      def command_args(query_path)
+        ds.query_command_args("--query=#{query_path}", "--output=#{export_path}", "--format=#{@format}")
+      end
+
+      def export_path
+        @gzip ? @path.sub(/\.gz\z/, '') : @path
+      end
+    end
+
+    def delete(table_name, **opts)
+      add TDTask::Delete.new(table_name, **opts)
+    end
+
+    class Delete < Action
+      def initialize(table_name, from:, to:)
+        @table = table_name
+        @from = from.to_time.to_i
+        @to = to.to_time.to_i
+      end
+
+      def run
+        td_result = ds.exec(source)
+      end
+
+      def command_args
+        ds.delete_command_args(@table, "--from #{@from}", "--to #{@to}")
+      end
+
+      def source
+        buf = StringIO.new
+        buf.puts command_args.join(' ')
+        buf.string
+      end
+    end
+  end
+
+end

data/lib/bricolage/variables.rb

@@ -0,0 +1,208 @@
+require 'bricolage/exception'
+
+module Bricolage
+
+  class Variables
+    def Variables.union(*vars_list)
+      new.tap {|result|
+        vars_list.each do |vars|
+          result.update vars
+        end
+      }
+    end
+
+    def Variables.define
+      new.tap {|vars|
+        yield vars
+      }
+    end
+
+    def initialize
+      @vars = {}
+    end
+
+    def inspect
+      "\#<#{self.class} #{@vars.inspect}>"
+    end
+
+    def resolved?
+      false
+    end
+
+    def each_variable(&block)
+      @vars.each_value(&block)
+    end
+
+    def [](name)
+      var = @vars[name] or raise ParameterError, "no such variable: #{name}"
+      var.value
+    end
+
+    def []=(name, value)
+      add Variable.new(name, value)
+    end
+
+    def keys
+      @vars.keys
+    end
+
+    def add(var)
+      @vars[var.name] = var
+    end
+
+    def update(vars)
+      vars.each_variable do |var|
+        add var
+      end
+    end
+
+    # resolve self recursively
+    def resolve
+      ResolvedVariables.define {|resolved|
+        @vars.each_value do |var|
+          resolved.add do_expand_variable(var, resolved, {})
+        end
+      }
+    end
+
+    def do_expand_variable(var, resolved, seen)
+      if seen[var.name]
+        cycle = (seen.keys + [var.name]).join(', ')
+        raise ParameterError, "recursive variable reference: #{var.name} (#{cycle})"
+      end
+      seen[var.name] = true
+      if var.resolved?
+        var
+      else
+        value = Variable.expand_string(var.value.to_s) {|name|
+          if resolved.bound?(name)
+            resolved[name]
+          elsif var2 = @vars[name]
+            res_var2 = do_expand_variable(var2, resolved, seen)
+            resolved.add res_var2
+            res_var2.value
+          else
+            raise ParameterError, "undefined variable in parameter #{var.name}: $#{name}"
+          end
+        }
+        ResolvedVariable.new(var.name, value)
+      end
+    end
+    private :do_expand_variable
+
+    def resolve_with(resolved)
+      raise "[BUG] unresolved variables given" unless resolved.resolved?
+      ResolvedVariables.define {|result|
+        each_variable do |var|
+          if var.resolved?
+            result.add var
+          else
+            val = resolved.expand(var.value.to_s)
+            result.add ResolvedVariable.new(var.name, val)
+          end
+        end
+      }
+    end
+  end
+
+  class ResolvedVariables
+    def ResolvedVariables.define
+      new.tap {|vars|
+        yield vars
+        vars.freeze
+      }
+    end
+
+    def initialize
+      @vars = {}
+    end
+
+    def inspect
+      "\#<#{self.class} #{@vars.inspect}>"
+    end
+
+    def resolved?
+      true
+    end
+
+    def freeze
+      @vars.freeze
+      super
+    end
+
+    def add(var)
+      raise "[BUG] unresolved variable: #{var.name}" unless var.resolved?
+      @vars[var.name] = var
+    end
+
+    def resolve_update(unresolved)
+      raise "[BUG] already resolved variables given" if unresolved.resolved?
+      unresolved.resolve_with(self).each_variable do |var|
+        add var
+      end
+    end
+
+    def bound?(name)
+      @vars.key?(name.to_s)
+    end
+
+    def [](name)
+      var = @vars[name.to_s] or raise ParameterError, "undefined parameter: #{name}"
+      var.value
+    end
+
+    def keys
+      @vars.keys
+    end
+
+    def each_variable(&block)
+      @vars.each_value(&block)
+    end
+
+    def expand(str)
+      Variable.expand_string(str) {|name| self[name] }
+    end
+
+    def bind_declarations(decls)
+      decls.each do |decl|
+        unless bound?(decl.name)
+          raise ParameterError, "script parameter not given: #{decl.name}"
+        end
+      end
+    end
+  end
+
+  class Variable
+    # generic variable extractor
+    def Variable.expand_string(str)
+      str.gsub(/\$(\w+)|\$\{(\w+)\}/) { yield ($1 || $2) }
+    end
+
+    def Variable.list(str)
+      str.scan(/\$(\w+)|\$\{(\w+)\}/).flatten.compact.uniq
+    end
+
+    def initialize(name, value)
+      @name = name
+      @value = value
+    end
+
+    attr_reader :name
+    attr_reader :value
+
+    def resolved?
+      false
+    end
+
+    def inspect
+      "\#<#{self.class} #{@name}=#{@value.inspect}>"
+    end
+  end
+
+  class ResolvedVariable < Variable
+    def resolved?
+      true
+    end
+  end
+
+end

data/lib/bricolage/version.rb

@@ -0,0 +1,4 @@
+module Bricolage
+  APPLICATION_NAME = 'Bricolage'
+  VERSION = '5.8.7'
+end

data/lib/bricolage.rb

@@ -0,0 +1,8 @@
+require 'bricolage/context'
+require 'bricolage/filesystem'
+require 'bricolage/job'
+require 'bricolage/rubyjobclass'
+require 'bricolage/datasource'
+require 'bricolage/eventhandlers'
+require 'bricolage/logger'
+require 'bricolage/exception'

data/libexec/sqldump

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+binary="$0.$(uname -s)"
+if ! [[ -x $binary ]]
+then
+    echo "$0: error: sqldump does not support $(uname -s)" 1>&2
+    exit 1
+fi
+exec "$binary" "$@"

data/test/all.rb

@@ -0,0 +1,3 @@
+Dir.glob("#{__dir__}/test_*.rb").each do |path|
+  load path
+end

data/test/home/config/development/database.yml

@@ -0,0 +1,57 @@
+sql: &redshift_parameters
+  type: psql
+  host: <%= ENV['RSPROXYHOST'] ? 'localhost' : (ENV['RSHOST'] || 'cookpad-dwh-201.cbzwjkf60uat.ap-northeast-1.redshift.amazonaws.com') %>
+  port: <%= ENV['RSPROXYPORT'] || ENV['RSPORT'] || 5439 %>
+  database: production
+  username: tabemirudev
+  pgpass: <%= user_home_relative_path '.pgpass' %>
+  encoding: utf8
+
+sql_dev:
+  <<: *redshift_parameters
+
+sql_prod:
+  <<: *redshift_parameters
+  username: tabemiru_batch
+
+sql_app:
+  type: psql
+  host: localhost   #cookpad-dwh-101.cbzwjkf60uat.ap-northeast-1.redshift.amazonaws.com
+  port: 5445
+  database: production
+  username: tabemiru_batch
+  pgpass: <%= user_home_relative_path '.pgpass' %>
+  encoding: utf8
+
+sql_app2:
+  type: psql
+  host: localhost
+  port: 5446
+  database: production
+  username: tabemiru_batch
+  pgpass: <%= user_home_relative_path '.pgpass' %>
+  encoding: utf8
+
+td:
+  type: td
+  database: logs
+  apikey: <%= password 'td_tabemiru' %>
+
+td_search_log:
+  type: td
+  database: search_log
+  apikey: <%= password 'td_tabemiru' %>
+
+s3:
+  type: s3
+  bucket: tabemiru-data.ap-northeast-1
+  prefix: "/dev"
+  s3cfg: <%= user_home_relative_path '.s3cfg' %>
+
+mysql:
+  type: mysql
+  host: db-main-slave-free-001
+  database: main
+  username: work_readonly
+  password: <%= password 'mysql_local_work_readonly' %>
+  encoding: utf8

data/test/home/config/development/password.yml

@@ -0,0 +1,2 @@
+td_tabemiru: "121/c1376bb5e3972fc4a4efc942c1df100f7bc156e0"
+mysql_local_work_readonly: "ckpdtest"

data/test/home/subsys/separated.job

@@ -0,0 +1 @@
+class: sql

data/test/home/subsys/separated.sql

@@ -0,0 +1 @@
+select 1;

data/test/home/subsys/unified.jobnet

@@ -0,0 +1 @@
+unified

data/test/home/subsys/unified.sql.job

@@ -0,0 +1,5 @@
+/*
+    class: sql
+*/
+
+select 1;

data/test/test_filesystem.rb

@@ -0,0 +1,19 @@
+require 'test/unit'
+require 'bricolage/filesystem'
+require 'pathname'
+require 'pp'
+
+module Bricolage
+  class TestFileSystem < Test::Unit::TestCase
+    fs = FileSystem.new("#{__dir__}/home", "test")
+    subfs = fs.subsystem('subsys')
+
+    test "FileSystem.job_file" do
+      path = subfs.job_file('unified')
+      assert_instance_of Pathname, path
+      assert_equal subfs.relative('unified.sql.job'), path
+
+      assert_equal subfs.relative('separated.job'), subfs.job_file('separated')
+    end
+  end
+end