terastream 0.1.0-java

data/lib/terastream/middleware/formatters/csv_builder.rb

@@ -0,0 +1,21 @@
+require "csv"
+
+module Terastream
+  class CSVBuilder < Terastream::Query::BaseFormatter
+    def default_output(&block)
+      CSV &block
+    end
+
+    def init
+      @app.headers
+    end
+
+    def call(&block)
+      row_data = []
+      @app.headers.each_with_index do |header, index|
+        row_data << @app.record_set.getString(index + 1)
+      end
+      row_data
+    end
+  end
+end

data/lib/terastream/middleware/formatters/json_builder.rb

@@ -0,0 +1,17 @@
+require "json"
+
+module Terastream
+  class JsonBuilder < Terastream::Query::BaseFormatter
+    def call(&block)
+      row_data = {}
+      @app.headers.each_with_index do |header, index|
+        row_data[header] = @app.record_set.getString(index + 1)
+      end
+      if block_given?
+        yield row_data.to_json
+      else
+        row_data.to_json
+      end
+    end
+  end
+end

data/lib/terastream/middleware/output/kafka.rb

@@ -0,0 +1,42 @@
+require 'poseidon'
+
+module Terastream
+  module Output
+    class Kafka
+      def initialize(options = {})
+        @options = options
+        @connection = Poseidon::Producer.new([connection], producer)
+      end
+
+      def <<(record)
+        @connection.send_messages [build_message(record)]
+      end
+
+      private
+
+      def build_message(record)
+        Poseidon::MessageToSend.new(topic, record)
+      end
+
+      def connection
+        "#{host}:#{port}"
+      end
+
+      def host
+        @options["host"] || "localhost"
+      end
+
+      def port
+        @options["port"] || "9092"
+      end
+
+      def producer
+        @options["producer"] || "terastream-producer"
+      end
+
+      def topic
+        @options["topic"] || "terastream"
+      end
+    end
+  end
+end

data/lib/terastream/middleware/output/kinesis.rb

@@ -0,0 +1,41 @@
+require "aws-sdk"
+
+module Terastream
+  module Output
+    class Kinesis
+      def initialize(options = {})
+        @options = options || {}
+        @connection = Aws::Kinesis::Client.new
+        @max_retries = 4
+      end
+
+      def <<(record)
+        put_record(record)
+      end
+
+      private
+
+      def put_record(data)
+        tries = @max_retries
+        begin
+          @connection.put_record(
+            stream_name: @options[:stream_name],
+            data: data,
+            partition_key: partition_key
+          )
+        rescue => e
+          tries -= 1
+          retry if tries > 0
+        end
+      end
+
+      def region
+        options[:region] || ENV["region"]
+      end
+
+      def partition_key
+        Time.now
+      end
+    end
+  end
+end

data/lib/terastream/middleware/output/redis.rb

@@ -0,0 +1,25 @@
+require 'redis'
+
+module Terastream
+  module Output
+    class Redis
+      def initialize(options = {})
+        @connection = ::Redis.new
+      end
+
+      def <<(record)
+        @connection.publish(:messages, record)
+      end
+
+      private
+
+      def host
+        @options["host"] || "localhost"
+      end
+
+      def port
+        @options["port"] || "9092"
+      end
+    end
+  end
+end

data/lib/terastream/middleware/output/s3.rb

@@ -0,0 +1,49 @@
+require "aws-sdk"
+require "securerandom"
+
+module Terastream
+  module Output
+    class S3
+      attr_accessor :options
+
+      def initialize(options = {})
+        @options = options
+        @temp = ""
+        @connection = Aws::S3::Client.new(region: region)
+        @records = []
+      end
+
+      def <<(record)
+        @temp += record.is_a?(Hash) || record.is_a?(String) ? as_json(record) : as_csv(record)
+      end
+
+      def complete!
+        tries = @max_retries
+        begin
+          @connection.put_object(
+            key: options[:key],
+            bucket: options[:bucket],
+            body: @temp
+          )
+        rescue => e
+          retries -= 1
+          retry if tries > 0
+        end
+      end
+
+      private
+
+      def as_json(record)
+        "#{record.to_json}\n"
+      end
+
+      def as_csv(record)
+        CSV.generate{ |csv| csv << record }
+      end
+
+      def region
+        options[:region] || ENV["region"] || "us-east-1"
+      end
+    end
+  end
+end

data/lib/terastream/query.rb

@@ -0,0 +1,31 @@
+module Terastream
+  class Query
+    attr_accessor :result_set, :statement
+
+    class << self
+      def build(connection)
+        new(connection)
+      end
+    end
+
+    def initialize(connection)
+      @connection = connection
+      @statement = connection.create_statement
+      @statement.setQueryTimeout(connection.config.timeout)
+      @executed = false
+      @result_set = []
+    end
+
+    def executed?
+      @executed
+    end
+
+    def execute(sql)
+      results = ResultSet.new(statement.execute_query(sql), @connection.formatter, @connection.output)
+      @executed = true
+      results
+    rescue  => e
+      raise Terastream::Errors::QueryError.new("Database exception: #{e.message}")
+    end
+  end
+end

data/lib/terastream/query/base_formatter.rb

@@ -0,0 +1,13 @@
+module Terastream
+  class Query
+    class BaseFormatter
+      def initialize(config = {})
+        @config = config
+      end
+
+      def app(env)
+        @app = env
+      end
+    end
+  end
+end

data/lib/terastream/query/records_builder.rb

@@ -0,0 +1,31 @@
+require "ostruct"
+
+module Terastream
+  class Query
+    class RecordsBuilder < Terastream::Query::BaseFormatter
+      def call
+        row_data = {}
+        @app.headers.each_with_index do |header, index|
+          row_data[header] = @app.record_set.getString(index + 1)
+        end
+        Record.new(row_data)
+      end
+    end
+
+    class Record
+      attr_reader :data
+
+      def initialize(data = {})
+        @data = OpenStruct.new(data)
+      end
+
+      def method_missing(name, *args, &block)
+        if data.respond_to?(name)
+          data.send(:name, *args, &block)
+        else
+          super
+        end
+      end
+    end
+  end
+end

data/lib/terastream/query/result_set.rb

@@ -0,0 +1,52 @@
+require "forwardable"
+
+module Terastream
+  class Query
+    class ResultSet
+      include Enumerable
+
+      extend Forwardable
+
+      attr_accessor :record_set, :headers, :formatter, :results, :output
+
+      def_delegators :@metadata, :column_count, :columns, :headers, :types
+
+      def initialize(record_set, formatter = Terastream::JsonBuilder.new, destination = nil)
+        @record_set = record_set
+        @metadata = ResultSetMetadata.new(record_set)
+        @formatter = formatter
+        @output = destination || default_output
+        @_results = nil
+        @formatter.app(self)
+      end
+
+      def results
+        @_results ||= build
+      end
+
+      def build
+        if formatter.respond_to?(:init)
+          @output << formatter.init
+        end
+
+        while row = @record_set.next
+          @output << formatter.call
+        end
+        @output.send(:complete!) if @output.respond_to?(:complete!)
+        @output if @output.instance_of?(Array)
+      end
+
+      def each(&block)
+        results.each do |result|
+          yield result
+        end
+      end
+
+      private
+
+      def default_output
+        formatter.respond_to?(:default_output) ? formatter.default_output : []
+      end
+    end
+  end
+end

data/lib/terastream/query/result_set_metadata.rb

@@ -0,0 +1,37 @@
+module Terastream
+  class Query
+    class ResultSetMetadata
+      def initialize(record_set)
+        @metadata = record_set.getMetaData()
+      end
+
+      def column_count
+        @column_count ||= @metadata.getColumnCount()
+      end
+
+      def columns
+        @columns ||= extract_metadata
+      end
+
+      def headers
+        @heders ||= columns.map{ |col| col[:name] }
+      end
+
+      def types
+        @types ||= columns.map{ |col| col[:type] }
+      end
+
+      private
+
+      def extract_metadata
+        (1..column_count).inject([]) do |data, index|
+          data << {
+            name: @metadata.getColumnName(index),
+            type: @metadata.getColumnType(index)
+          }
+          data
+        end
+      end
+    end
+  end
+end

data/lib/terastream/query/type_map.rb

@@ -0,0 +1,13 @@
+module Terastream
+  class TypeMap
+    class << self
+      def map(value)
+        if [1, -9, 12, -15, 91].include?(value)
+          :getString
+        else
+          :getObject
+        end
+      end
+    end
+  end
+end

data/lib/terastream/version.rb

@@ -0,0 +1,3 @@
+module Terastream
+  VERSION = "0.1.0"
+end

data/terastream.gemspec

@@ -0,0 +1,41 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'terastream/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "terastream"
+  spec.version       = Terastream::VERSION
+  spec.authors       = ["Keene, Ramin"]
+  spec.email         = ["Ramin.Keene@nordstrom.com"]
+
+  spec.summary       = %q{ Stream query results out of teradata data warehouse}
+  spec.description   = %q{ Utility gem to stream teradata results out of the teradata data warehouse, supporting adaptors for various formats and destinations}
+  spec.homepage      = "http://github.com/Nordstrom/terastream"
+  spec.license       = "MIT"
+
+  # Prevent pushing this gem to RubyGems.org by setting 'allowed_push_host', or
+  # delete this section to allow pushing this gem to any host.
+  if spec.respond_to?(:metadata)
+    spec.metadata['allowed_push_host'] = "https://rubygems.org"
+  else
+    raise "RubyGems 2.0 or newer is required to protect against public gem pushes."
+  end
+
+  spec.platform = 'java'
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "bin"
+  spec.executables  << 'terasql'
+
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "aws-sdk", "~> 2"
+  spec.add_dependency "json", "~> 1.8.0"
+  spec.add_dependency "poseidon", "~> 0.0.5"
+  spec.add_dependency "redis", "~> 3.2.1"
+
+  spec.add_development_dependency "bundler", "~> 1.9"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.3.0"
+end