tensor_stream-opencl 0.1.0

data/lib/tensor_stream/opencl/nn_ops.rb

@@ -0,0 +1,191 @@
+module TensorStream
+  module OpenCLHelpers
+    # Collection of math functions for interfacing with OpenCL kernels
+    module NNOps
+      def NNOps.included(klass)
+        klass.class_eval do
+
+          # Fast in place multiply subtract assign
+          register_op :apply_gradient_descent do |_context, tensor, inputs|
+            _target_var, learning_rate, delta = inputs
+
+            assign = tensor.inputs[0] || tensor
+
+            assign.buffer.dirty = true # force buffer copy when variable is read externally
+            output_buffer = assign.buffer
+
+            m, n = output_buffer.shape
+            work_group = [m || 1, n || 1]
+            cl_m = OpenCL::Int1.new(m || 1)
+            cl_n = OpenCL::Int1.new(n || 1)
+
+            event_wait_list = build_event_wait_list([assign.buffer, learning_rate, delta])
+            method_call = :"apply_gradient_#{output_buffer.data_type}"
+            event = _cl_program("apply_gradient", dtype: output_buffer.data_type).send(method_call, _opencl_queue, work_group, cl_m, cl_n, delta.cl_buffer, learning_rate.cl_buffer, output_buffer.cl_buffer, event_wait_list: event_wait_list)
+            output_buffer.op = event
+            output_buffer
+          end
+
+          # updates for gradient descent with momentum
+          register_op :apply_momentum do |_context, tensor, inputs|
+            target_var, momentum_var, learning_rate, grad, momentum = inputs
+
+            assign = tensor.inputs[0] || tensor
+            assign_acc = tensor.inputs[1]
+            assign.buffer.dirty = true # force buffer copy when variable is read externally
+            assign_acc.buffer.dirty = true # force buffer copy when variable is read externally
+
+            output_buffer = assign.buffer
+
+            m, n = output_buffer.shape
+            work_group = [m || 1, n || 1]
+            cl_m = OpenCL::Int1.new(m || 1)
+            cl_n = OpenCL::Int1.new(n || 1)
+
+            event_wait_list = build_event_wait_list([assign.buffer, assign_acc.buffer, learning_rate, grad, momentum])
+            method_call = :"apply_momentum_#{output_buffer.data_type}"
+            event = _cl_program("apply_momentum", nesterov: tensor.options[:use_nesterov], dtype: output_buffer.data_type).
+                        send(method_call, _opencl_queue, work_group, cl_m, cl_n, grad.cl_buffer,
+                            learning_rate.cl_buffer, momentum.cl_buffer, output_buffer.cl_buffer,
+                            assign_acc.buffer.cl_buffer, event_wait_list: event_wait_list)
+            output_buffer.op = event
+            assign_acc.buffer.op = event
+            output_buffer
+          end
+
+          register_op :apply_adadelta do |context, tensor, inputs|
+          end
+
+          # Adam optimization algorithm
+          register_op :apply_adam do |_context, tensor, inputs|
+            _target_var, _m, _v, beta1_power, beta2_power, lr_t, beta1_t, beta2_t, epsilon_t, grad = inputs
+
+            assign = tensor.inputs[0] || tensor
+            assign_m = tensor.inputs[1]
+            assign_v = tensor.inputs[2]
+
+            # mark variable buffers as dirty
+            assign.buffer.dirty = true # force buffer copy when variable is read externally
+            assign_m.buffer.dirty = true # force buffer copy when variable is read externally
+            assign_v.buffer.dirty = true # force buffer copy when variable is read externally
+
+            output_buffer = assign.buffer
+
+            m, n = output_buffer.shape
+            work_group = [m || 1, n || 1]
+            cl_m = OpenCL::Int1.new(m || 1)
+            cl_n = OpenCL::Int1.new(n || 1)
+
+            event_wait_list = build_event_wait_list(inputs)
+            method_call = :"apply_adam_#{output_buffer.data_type}"
+            event = _cl_program("apply_adam", dtype: output_buffer.data_type)
+                                .send(method_call, _opencl_queue, work_group, cl_m, cl_n,
+                                      grad.cl_buffer,
+                                      lr_t.cl_buffer,
+                                      beta1_power.cl_buffer,
+                                      beta2_power.cl_buffer,
+                                      beta1_t.cl_buffer,
+                                      beta2_t.cl_buffer,
+                                      epsilon_t.cl_buffer,
+                                      assign_m.buffer.cl_buffer,
+                                      assign.buffer.cl_buffer,
+                                      assign_v.buffer.cl_buffer,
+                                      event_wait_list: event_wait_list)
+            output_buffer.op = event
+            assign_m.buffer.op = event
+            assign_v.buffer.op = event
+            output_buffer
+          end
+
+          register_op :softmax do |_context, tensor, inputs|
+            a = inputs[0]
+            event_wait_list = build_event_wait_list(inputs)
+            dtype = tensor.data_type
+            output_buffer = _create_result_buffer(tensor.data_type, a.shape, tensor.name)
+
+            m, n = a.shape
+            work_group = [m]
+            n = m if n.nil?
+            cl_n = OpenCL::Int1.new(n || 1)
+
+            event = _cl_program("softmax", dtype: dtype).send(:"softmax_#{dtype}", _opencl_queue, work_group, cl_n, a.cl_buffer, output_buffer.cl_buffer, event_wait_list: event_wait_list)
+            output_buffer.op = event
+            output_buffer
+          end
+
+          register_op :log_softmax do |_context, tensor, inputs|
+            a = inputs[0] # logits
+            event_wait_list = build_event_wait_list(inputs)
+            dtype = tensor.data_type
+            output_buffer = _create_result_buffer(tensor.data_type, a.shape, tensor.name)
+
+            m, n = a.shape
+            work_group = [m]
+            n = m if n.nil?
+            cl_n = OpenCL::Int1.new(n || 1)
+
+            event = _cl_program("log_softmax", dtype: dtype).send(:"log_softmax_#{dtype}", _opencl_queue, work_group, cl_n, a.cl_buffer, output_buffer.cl_buffer, event_wait_list: event_wait_list)
+            output_buffer.op = event
+            output_buffer
+          end
+
+          register_op :softmax_cross_entropy_with_logits_v2 do |context, tensor, inputs|
+            a = inputs[0] # logits
+            b = inputs[1] # labels
+            event_wait_list = build_event_wait_list(inputs)
+            dtype = tensor.data_type
+            output_buffer = _create_result_buffer(tensor.data_type, a.shape, tensor.name)
+            output_buffer_backprop = _create_result_buffer(tensor.data_type, a.shape, "#{tensor.name}_2")
+            rank = a.shape.size - 1
+            m, n = a.shape
+            work_group = [m]
+            n = m if n.nil?
+            cl_n = OpenCL::Int1.new(n || 1)
+
+            event = _cl_program("softmax_cross", dtype: dtype).send(:"softmax_cross_#{dtype}", _opencl_queue, work_group, cl_n, a.cl_buffer, b.cl_buffer,
+                                 output_buffer.cl_buffer, output_buffer_backprop.cl_buffer, event_wait_list: event_wait_list)
+            output_buffer.op = event
+            output_buffer_backprop.op = event
+
+            loss = reduction(context, tensor, output_buffer, rank, :sum)
+            TensorStream::Evaluator::OutputGroup.new([loss, output_buffer_backprop],  [tensor.inputs[0].data_type, tensor.inputs[0].data_type])
+          end
+
+          register_op :softmax_cross_entropy_with_logits_v2_grad do |_context, tensor, inputs|
+            a = inputs[0] # logits
+            b = inputs[1] # labels
+            c = inputs[2] # grads
+            event_wait_list = build_event_wait_list(inputs)
+            dtype = tensor.data_type
+            output_buffer = _create_result_buffer(tensor.data_type, a.shape, tensor.name)
+
+            m, n = a.shape
+            work_group = [m]
+            n = m if n.nil?
+            cl_n = OpenCL::Int1.new(n || 1)
+
+            event = _cl_program("softmax_cross_grad", dtype: dtype).send(:"softmax_cross_grad_#{dtype}", _opencl_queue, work_group, cl_n, a.cl_buffer, b.cl_buffer, c.cl_buffer, output_buffer.cl_buffer, event_wait_list: event_wait_list)
+            output_buffer.op = event
+            output_buffer
+          end
+
+          register_op :softmax_grad do |_context, tensor, inputs|
+            a, grad = inputs
+
+            event_wait_list = build_event_wait_list(inputs)
+            dtype = tensor.data_type
+            output_buffer = _create_result_buffer(tensor.data_type, a.shape, tensor.name)
+
+            m, n = a.shape
+            work_group = [m]
+            n = m if n.nil?
+            cl_n = OpenCL::Int1.new(n || 1)
+            event = _cl_program('softmax_grad', dtype: dtype, size: n).send(:"softmax_grad_#{dtype}", _opencl_queue, work_group, cl_n, a.cl_buffer, grad.cl_buffer, output_buffer.cl_buffer, event_wait_list: event_wait_list)
+            output_buffer.op = event
+            output_buffer
+          end
+        end
+      end
+    end
+  end
+end

data/lib/tensor_stream/opencl/opencl_buffer.rb

@@ -0,0 +1,35 @@
+module TensorStream
+  # Buffer used by the OpenCL evaluator
+  class OpenCLBuffer < Buffer
+    include ArrayOpsHelper
+
+    attr_accessor :shape, :buffer, :cl_buffer, :op
+
+    def initialize(data_type:, shape:, buffer:, cl_buffer:, op: nil, name: nil)
+      @data_type = data_type
+      @shape = shape
+      @buffer = buffer
+      @cl_buffer = cl_buffer
+      @name = name
+      @op = op
+    end
+
+    def to_ruby
+      return [] if buffer.empty?
+
+      if dirty
+        op.command_queue.enqueue_read_buffer(cl_buffer, buffer, event_wait_list: [op].compact)
+        op.command_queue.finish
+        self.dirty = false
+      end
+
+      if shape.empty?
+        return buffer[0] != 0 if data_type == :boolean
+        return buffer[0]
+      end
+
+      result = buffer.reshape(*shape.map(&:to_i).reverse).to_a
+      data_type == :boolean ? process_function_op(result, ->(a, _b) { a != 0 }) : result
+    end
+  end
+end

data/lib/tensor_stream/opencl/opencl_device.rb

@@ -0,0 +1,5 @@
+module TensorStream
+  class OpenclDevice < TensorStream::Device
+    attr_accessor :native_device
+  end
+end

data/lib/tensor_stream/opencl/opencl_evaluator.rb

@@ -0,0 +1,933 @@
+require 'tensor_stream/evaluator/operation_helpers/random_gaussian'
+require 'tensor_stream/evaluator/operation_helpers/array_ops_helper'
+require 'tensor_stream/evaluator/operation_helpers/math_helper'
+require 'tensor_stream/evaluator/buffer'
+require 'tensor_stream/opencl/opencl_buffer'
+require 'tensor_stream/opencl/opencl_template_helper'
+require 'tensor_stream/device'
+require 'tensor_stream/evaluator/opencl/opencl_device'
+require 'opencl_ruby_ffi'
+require 'narray_ffi'
+require 'tensor_stream/evaluator/base_evaluator'
+require 'tensor_stream/opencl/math_ops'
+require 'tensor_stream/opencl/nn_ops'
+require 'tensor_stream/helpers/op_helper'
+
+module TensorStream
+  module Evaluator
+    class FullEvalNotPossible < RuntimeError
+    end
+
+    # Errors during graph evaluation
+    class EvaluatorExcecutionException < RuntimeError
+      attr_reader :tensor
+
+      def initialize(exception, tensor)
+        @exception = exception
+        @tensor = tensor
+      end
+
+      def wrapped_exception
+        @exception
+      end
+    end
+
+    ## PURE ruby evaluator used for testing and development
+    class OpenclEvaluator < BaseEvaluator
+      attr_accessor :retain
+      attr_reader :opencl_device
+
+      include TensorStream::OpHelper
+      include TensorStream::ArrayOpsHelper
+      include TensorStream::MathHelper
+      include TensorStream::OpenCLHelpers::MathOps
+      include TensorStream::OpenCLHelpers::NNOps
+
+      def initialize(session, device, thread_pool: nil, log_intermediates: false)
+        super
+        _create_opencl_context(device.native_device)
+        @opencl_device = device.native_device
+        create_command_queue
+      end
+
+      def self.query_supported_devices
+        devices = query_devices_with_score
+        devices.sort { |a| a[1] }.reverse.map do |d|
+          opencl_to_device(d)
+        end
+      end
+
+      def self.fetch_device(query = [])
+        devices = query_devices_with_score
+        platform_devices = devices.select { |d| d[0].platform.to_s.tr(' ', '_').downcase =~ /#{query[0].downcase}/ }
+        opencl_to_device(platform_devices[[query[1].to_i, platform_devices.size - 1].min])
+      end
+
+      def self.opencl_to_device(d)
+        device = d[0]
+        index = d[3]
+        platform_name = device.platform.name.tr(' ', '_').downcase
+        uri = [platform_name, index].join(':')
+
+        device_type = device.type.to_s == 'GPU' ? :gpu : :cpu
+
+        OpenclDevice.new(uri, device_type, self).tap do |devide|
+          devide.native_device = device
+        end
+      end
+
+      ##
+      # Select the best device available in the system for this evaluator
+      def self.default_device
+        devices = OpenclEvaluator.query_devices_with_score
+        device = devices.sort { |a| a[1] }.reverse.first
+        opencl_to_device(device)
+      end
+
+      # opencl evaluator main entrypoint
+      def run(tensor, execution_context)
+        read_final_result(complete_eval(tensor, execution_context))
+      end
+
+      def run_with_buffer(tensor, context, execution_context)
+        @context = context
+        @context[:_cache][:_cl_buffers] ||= {} if context[:_cache]
+
+        if tensor.is_a?(Array)
+          tensor.collect do |t|
+            value = run(t, execution_context)
+            Buffer.new(data_type: t.data_type, buffer: value)
+          end
+        else
+          value = run(tensor, execution_context)
+          Buffer.new(data_type: tensor.data_type, buffer: value)
+        end
+      end
+
+      # buffer comes from non-opencl evaluator
+      def convert_from_buffer(tensor, result)
+        if result.buffer.is_a?(TensorStream::Evaluator::OutputGroup)
+          converted_outputs = result.buffer.outputs.zip(result.buffer.data_types).map { |output, data_type| convert_to_opencl([output].flatten, shape_eval(output), data_type: data_type, name: tensor.name) }
+          TensorStream::Evaluator::OutputGroup.new(converted_outputs, result.buffer.data_types)
+        else
+          convert_to_opencl([result.buffer].flatten, shape_eval(result.buffer), data_type: result.data_type, name: tensor.name)
+        end
+      end
+
+      def enqueue_buffer_read(tensor, context)
+        buffer = _run(tensor, context)
+        if buffer.is_a?(Array)
+          buffer = buffer.collect do |b|
+            next b if b.buffer.size.zero?
+            _opencl_queue.enqueue_read_buffer(b.cl_buffer, b.buffer, event_wait_list: build_event_wait_list([b]))
+            b
+          end
+        else
+          return buffer.outputs[0] if buffer.is_a?(OutputGroup)
+          return buffer if buffer.nil?
+          return [] if buffer.buffer.nil?
+          return buffer if buffer.buffer.size.zero?
+          _opencl_queue.enqueue_read_buffer(buffer.cl_buffer, buffer.buffer, event_wait_list: build_event_wait_list([buffer]))
+          buffer
+        end
+      end
+
+      def complete_eval(tensor, context)
+        buffer = enqueue_buffer_read(tensor, context)
+        _opencl_queue.finish
+        buffer
+      end
+
+      def self.query_devices_with_score
+        OpenCL.platforms.flat_map do |p|
+          p.devices.select { |d| d.available > 0 }.each_with_index.collect do |d, index|
+            score = 0
+            if d.type.to_s == 'CPU'
+              score += 1
+            elsif d.type.to_s == 'GPU'
+              score += 4
+            end
+
+            score += 1000 if d.platform.name == 'NVIDIA CUDA'
+
+            score += d.max_compute_units
+            score += d.max_clock_frequency
+
+            [d, score, p.name, index]
+          end
+        end
+      end
+
+      protected
+
+      def prepare_input(tensor, context, options = {})
+        return nil unless tensor
+        tensor = resolve_placeholder(tensor)
+        if options[:noop]
+          tensor
+        elsif options[:buffer]
+          complete_eval(tensor, context)
+        elsif options[:complete]
+          read_final_result(complete_eval(tensor, context))
+        else
+          _run(tensor, context)
+        end
+      end
+
+      # read result from opencl and convert to ruby
+      def read_final_result(buffer)
+        return buffer.map { |b| read_final_result(b) } if buffer.is_a?(Array)
+        return nil if buffer.nil?
+
+        buffer.to_ruby
+      end
+
+      def _create_opencl_context(opencl_device)
+        @opencl_context = OpenCL.create_context(opencl_device)
+      end
+
+      def create_command_queue
+        supported_proprties = opencl_device.queue_properties.names
+
+        properties = []
+        properties << OpenCL::CommandQueue::PROFILING_ENABLE if supported_proprties.include?('PROFILING_ENABLE')
+        properties << OpenCL::CommandQueue::OUT_OF_ORDER_EXEC_MODE_ENABLE if supported_proprties.include?('OUT_OF_ORDER_EXEC_MODE_ENABLE')
+        @command_queue = _opencl_context.create_command_queue(opencl_device, properties: properties)
+      end
+
+      def _opencl_context
+        @opencl_context
+      end
+
+      def _opencl_queue
+        @command_queue
+      end
+
+      def cl_template_path(kernel, extension)
+        File.join(File.dirname(__FILE__), 'kernels', "#{kernel}.#{extension}")
+      end
+
+      def _cl_program(kernel, args = {})
+        suffix = args.collect { |k, v| "#{k}.#{escape_arg_content(v)}" }.join('.')
+        @context[:_cache]["_opencl_kernel_#{kernel}.#{suffix}:#{object_id}"] ||= begin
+          filename = %w[cl.erb cl].map { |ext| cl_template_path(kernel, ext) }.find { |n| File.exist?(n) }
+          raise "opencl kernel template for #{kernel} has not yet been defined" if filename.nil?
+          source = File.read(filename)
+          source = OpenclTemplateHelper.new(source).generate(args)
+          # File.write("/tmp/#{kernel}.#{suffix}.cl", source)
+          program = _opencl_context.create_program_with_source(source)
+          program.build
+        rescue OpenCL::Error::BUILD_PROGRAM_FAILURE => e
+          puts "OpenCL Compile error: #{program.build_log}"
+          raise e
+        end
+      end
+
+      def escape_arg_content(value)
+        return value.tr(' ','_') if value.is_a?(String)
+        return value.join('-') if value.is_a?(Array)
+
+        value
+      end
+
+      def _run(tensor, execution_context)
+        return tensor if tensor.is_a?(OpenCLBuffer)
+        return tensor.map { |t| _run(t, execution_context) } if tensor.is_a?(Array) && !tensor.size.empty? && tensor[0].is_a?(Tensor)
+
+        tensor = tensor.call if tensor.is_a?(Proc)
+
+        child_context = execution_context.dup
+        res = if tensor.is_a?(Operation)
+                if !self.class.ops.include?(tensor.operation.to_sym)
+                  result = @session.delegate_to_evaluator(tensor, @context, execution_context)
+                  convert_from_buffer(tensor, result)
+                else
+                  eval_operation(tensor, child_context)
+                end
+              elsif tensor.is_a?(Variable)
+                eval_variable(tensor, child_context)
+              elsif tensor.is_a?(Placeholder)
+                resolve_placeholder(tensor, child_context)
+              else
+                eval_tensor(tensor, child_context)
+              end
+        execution_context.deep_merge!(returns: child_context[:returns])
+        res
+      end
+
+      def eval_variable(tensor, _child_context)
+        raise "variable #{tensor.name} not initalized" if tensor.value.nil? && (tensor.buffer.nil? || !tensor.buffer.dirty)
+        tensor.buffer = wrap_opencl(tensor, name: tensor.name) if tensor.buffer.nil?
+        tensor.buffer
+      end
+
+      register_op :no_op do |_context, _tensor, _inputs|
+      end
+
+      register_op :cond, noop: true do |context, tensor, inputs|
+        pred = complete_eval(tensor.options[:pred], context)
+
+        if all_true?(pred.buffer)
+          complete_eval(inputs[0], context)
+        else
+          complete_eval(inputs[1], context)
+        end
+      end
+
+      register_op :identity do |context, tensor, inputs|
+        if tensor.inputs.size > 1
+          tensor.inputs[1..inputs.size].each { |input| complete_eval(input, context) }
+        end
+        inputs[0]
+      end
+
+      register_op :assign, noop: true do |context, tensor, inputs|
+        assign_var(tensor, inputs[1], context)
+      end
+
+      register_op :assign_add do |context, tensor, inputs|
+        value = execute_2_operand_func('add', tensor, inputs[0], inputs[1], context)
+        assign_var(tensor, value, context)
+      end
+
+      register_op :assign_sub do |context, tensor, inputs|
+        value = execute_2_operand_func('sub', tensor, inputs[0], inputs[1], context)
+        assign_var(tensor, value, context)
+      end
+
+      register_op :variable, noop: true do |context, tensor, inputs|
+        variable = tensor.inputs[0]
+        raise "variable #{tensor.name} not initalized" if variable.value.nil? && (variable.buffer.nil? || !variable.buffer.dirty)
+        variable.buffer = wrap_opencl(variable, name: variable.name) if variable.buffer.nil?
+        variable.buffer
+      end
+
+      %i[less less_equal greater greater_equal equal not_equal logical_and].each do |op|
+        register_op op, noop: true do |context, tensor, inputs|
+          execute_2_operand_func(op.to_s, tensor, inputs[0], inputs[1], context, 'cond')
+        end
+      end
+
+      register_op :expand_dims, buffer: true do |_context, tensor, inputs|
+        axis = inputs[1].buffer[0]
+        shape = inputs[0].shape.dup
+        axis = -axis if axis == shape.size
+        new_shape = shape.insert(axis, 1).compact
+        new_buf = inputs[0].buffer.reshape(*new_shape.reverse)
+        convert_to_opencl(new_buf, new_shape, data_type: inputs[0].data_type, name: tensor.name)
+      end
+
+      register_op :fill, buffer: true do |_context, tensor, inputs|
+        shape = inputs[0]
+        value = inputs[1]
+
+        narray_size = shape.buffer.to_a.reduce(:*) || 1
+        cl_buffer = get_cached_buffer(tensor.name, shape.buffer.to_a)
+
+        buffer = if cl_buffer
+                   cl_buffer.buffer
+                 else
+                   allocate_narray_for_type(tensor.data_type, narray_size)
+                 end
+
+        buffer.fill!(value.buffer[0])
+        convert_to_opencl(buffer, shape.buffer.to_a, data_type: tensor.data_type, name: tensor.name)
+      end
+
+      register_op :where, noop: true do |context, tensor, inputs|
+        pred = tensor.options[:pred]
+        execute_cond_func('where', tensor, pred, inputs[0], inputs[1], context)
+      end
+
+      register_op :cast do |_context, tensor, inputs|
+        a = inputs[0]
+        if a.data_type != tensor.data_type
+          buffer = _create_result_buffer(tensor.data_type, a.shape, tensor.name)
+          m, n = a.shape
+          cl_m = OpenCL::Int1.new(m || 1)
+          cl_n = OpenCL::Int1.new(n || 1)
+          work_group = [m || 1, n || 1]
+          event_wait_list = build_event_wait_list(inputs)
+          buffer.op = _cl_program("cast", source_dt: a.data_type, target_dt: tensor.data_type).cast(_opencl_queue, work_group, cl_m, cl_n, a.cl_buffer, buffer.cl_buffer, event_wait_list: event_wait_list)
+          buffer
+        else
+          a
+        end
+      end
+
+      register_op :stack do |_context, tensor, inputs|
+        axis = tensor.options[:axis] || 0
+        shape = inputs[0].shape
+        rank = shape.size + 1
+        elem_size = shape.empty? ? 1 : shape.reduce(:*)
+
+        new_shape = [inputs.size]
+        shape.inject(new_shape) { |ns, s| ns << s }
+
+        divisors = new_shape.dup.drop(1).reverse.inject([1]) do |a, s|
+          a << s * a.last
+        end.reverse
+
+        axis = rank + axis if axis < 0
+        rotated_shape = Array.new(axis + 1) { new_shape.shift }
+        new_shape = rotated_shape.rotate! + new_shape
+
+        output_buffer = _create_result_buffer(tensor.data_type, new_shape, tensor.name)
+        multipliers = new_shape.dup.drop(1).reverse.inject([1]) do |a, s|
+          a << s * a.last
+        end.reverse
+
+        cl_n = OpenCL::Int1.new(elem_size)
+        work_group = [elem_size]
+        event_wait_list = build_event_wait_list(inputs)
+        ops = inputs.each_with_index.map do |input, index|
+          cl_index = OpenCL::Int1.new(index)
+          _cl_program("pack", data_type: tensor.data_type, divisors: divisors, multipliers: multipliers, axis: axis).pack(_opencl_queue, work_group, cl_n, cl_index, input.cl_buffer, output_buffer.cl_buffer, event_wait_list: event_wait_list)
+        end
+        output_buffer.op = ops
+        output_buffer
+      end
+
+      register_op :check_numerics, noop: true do |context, tensor, inputs|
+        a = complete_eval(inputs[0], context)
+        name = tensor.options[:name]
+
+        a.buffer.each do |input|
+          raise TensorStream::InvalidArgumentError, "#{name} Invalid Argument" if input.nan? || input.infinite?
+        end
+        a
+      end
+
+      register_op :broadcast_transform do |context, tensor, inputs|
+        a, b = inputs
+
+        if a.shape == b.shape
+          [a, b]
+        else
+          input_a = read_final_result(complete_eval(a, context))
+          input_b = read_final_result(complete_eval(b, context))
+          b_a, b_b = broadcast(input_a, input_b)
+          [wrap_opencl(b_a, data_type: a.data_type, name: "#{tensor.name}_a"),
+           wrap_opencl(b_b, data_type: a.data_type, name: "#{tensor.name}_b")]
+        end
+      end
+
+      register_op :print do |context, tensor, inputs|
+        a, b = inputs
+        input_b = complete_eval(b, context)
+        input_b = read_final_result(input_b)
+        puts "#{tensor.options.fetch(:message, '')} #{input_b}"
+        a
+      end
+
+      register_op :rank do |_context, tensor, inputs|
+        wrap_opencl(inputs[0].shape.size, data_type: tensor.data_type, name: tensor.name)
+      end
+
+      register_op :stop_gradient do |_context, _tensor, inputs|
+        inputs[0]
+      end
+
+      register_op :slice, noop: true do |context, tensor, inputs|
+        input_a = complete_eval(inputs[0], context)
+        input_b = read_final_result(complete_eval(inputs[1], context))
+        size = tensor.options[:size]
+
+        slice_param = input_b.zip(size).collect { |x, y| x..x + y - 1 }.reverse
+
+        new_buf = input_a.buffer.reshape(*input_a.shape.reverse)
+        sliced = new_buf.slice[*slice_param]
+        convert_to_opencl(sliced.flatten, sliced.shape.reverse, data_type: inputs[0].data_type, name: tensor.name)
+      end
+
+      register_op :transpose, buffer: true do |_context, tensor, inputs|
+        t_param = Array.new(inputs[0].shape.size) { |index| index }.reverse
+
+        if inputs[0].shape.size == 2 && inputs[1].nil?
+          transposed = inputs[0].buffer.reshape(*inputs[0].shape.reverse).transpose(*t_param)
+          res = convert_to_opencl(transposed.flatten, transposed.shape.reverse, data_type: inputs[0].data_type, name: tensor.name)
+          res
+        else
+          rank = inputs[0].shape.size
+          perm = inputs[1].nil? ? (0...rank).to_a.reverse : inputs[1].buffer
+          new_shape = perm.map { |p| inputs[0].shape[p] }.to_a
+          output_buffer = _create_result_buffer(tensor.data_type, new_shape, tensor.name)
+          transpose_with_perm(inputs[0].buffer, output_buffer.buffer, inputs[0].shape, new_shape, perm)
+
+          write_op = _opencl_queue.enqueue_write_buffer(output_buffer.cl_buffer, output_buffer.buffer)
+          output_buffer.op = write_op
+          output_buffer
+        end
+      end
+
+      register_op :index, noop: true do |context, tensor, inputs|
+        a = _run(inputs[0], context)
+        index = read_final_result(_run(inputs[1], context))
+
+        if a.is_a?(OutputGroup)
+          a.outputs[index]
+        elsif a.is_a?(Array)
+          a[index]
+        else
+          new_shape = a.shape.dup
+          new_shape.shift
+          input_a = read_final_result(a)
+          convert_to_opencl(input_a[index], new_shape, data_type: a.data_type, name: tensor.name)
+        end
+      end
+
+      register_op :broadcast_gradient_args, buffer: true do |_context, tensor, inputs|
+        rx, ry = get_broadcast_gradient_args(inputs[0].buffer.to_a, inputs[1].buffer.to_a)
+        OutputGroup.new([wrap_opencl(rx, data_type: :int32, name: tensor.name), wrap_opencl(ry, data_type: :int32, name: "#{tensor.name}:1")], tensor.inputs.map(&:data_type))
+      end
+
+      register_op :shape do |_context, tensor, inputs|
+        wrap_opencl(inputs[0].shape, name: tensor.name, data_type: tensor.data_type)
+      end
+
+      register_op :reshape, buffer: true do |_context, tensor, inputs|
+        arr = inputs[0]
+        new_shape = read_final_result(inputs[1])
+
+        shape = if new_shape.size.zero? && arr.buffer.size == 1
+                  new_shape
+                else
+                  TensorShape.fix_inferred_elements(new_shape, arr.buffer.size)
+                end
+
+        convert_to_opencl(arr.buffer, shape, data_type: arr.data_type, name: tensor.name)
+      end
+
+      register_op :flow_group do |context, _tensor, inputs|
+        _opencl_queue.finish
+        nil
+      end
+
+      register_op :size do |_context, tensor, inputs|
+        wrap_opencl(inputs[0].buffer.size, name: tensor.name, data_type: tensor.options[:out_type] || :int32)
+      end
+
+      def eval_operation(tensor, child_context)
+        cache_key = "#{tensor.graph.object_id}_opencl_#{tensor.name}:#{object_id}"
+        return @context[:_cache][cache_key] if @context[:_cache].key?(cache_key)
+        return @context[cache_key] if @context.key?(cache_key)
+        # puts "opencl: #{tensor.name}"
+        invoke(tensor, child_context).tap do |result|
+          if tensor.breakpoint
+            a = resolve_placeholder(tensor.inputs[0], child_context) if tensor.inputs && tensor.inputs[0]
+            b = resolve_placeholder(tensor.inputs[1], child_context) if tensor.inputs && tensor.inputs[1]
+            a = read_final_result(complete_eval(a, child_context))
+            b = read_final_result(complete_eval(b, child_context))
+            result = read_final_result(complete_eval(result, child_context))
+
+            tensor.breakpoint.call(tensor, a, b, result)
+          end
+          if @log_intermediates
+            @context[:compute_history] << {
+              name: tensor.name,
+              type: tensor.data_type,
+              shape: shape_eval(result),
+              source: tensor.source,
+              description: tensor.to_math(true, 1),
+              value: result
+            }
+          end
+          @context[cache_key] = result
+          @context[:_cache][cache_key] = result if tensor.is_const
+        end
+      rescue EvaluatorExcecutionException => e
+        _opencl_queue.finish # dump queue
+        raise e, "error #{e.message} while evaluating #{tensor.name} : #{tensor.to_math(true, 1)} defined at #{tensor.source}"
+      rescue TensorStreamError => e
+        _opencl_queue.finish # dump queue
+        raise e, "error #{e.message} while evaluating #{tensor.name} : #{tensor.to_math(true, 1)} defined at #{tensor.source}"
+      rescue StandardError => e
+        _opencl_queue.finish # dump queue
+        puts e.message
+        puts e.backtrace.join("\n")
+
+        # shape_a = a.shape.shape if a
+        # shape_b = b.shape.shape if b
+        # dtype_a = a.data_type if a
+        # dtype_b = b.data_type if b
+        # a = complete_eval(a, child_context)
+        # b = complete_eval(b, child_context)
+        # puts "name: #{tensor.given_name}"
+        # # puts "op: #{tensor.to_math(true, 1)}"
+        # puts "A #{shape_a} #{dtype_a}: #{a}" if a
+        # puts "B #{shape_b} #{dtype_b}: #{b}" if b
+        # dump_intermediates if @log_intermediates
+        # File.write('/home/jedld/workspace/tensor_stream/samples/error.graphml', TensorStream::Graphml.new.get_string(tensor, @session))
+
+        # File.write('/Users/josephemmanueldayo/workspace/gradients.graphml', TensorStream::Graphml.new.get_string(tensor, @session))
+        raise EvaluatorExcecutionException.new(e, tensor), "error #{e.message} while evaluating #{tensor.name} : #{tensor.to_math(true, 1)} defined at #{tensor.source}"
+      end
+
+      def eval_tensor(tensor, child_context)
+        return tensor unless tensor.is_a?(Tensor)
+
+        cache_key = "#{tensor.graph.object_id}_opencl_#{tensor.name}:#{object_id}"
+        return @context[cache_key] if @context.key?(cache_key)
+        return @context[:_cache][cache_key] if tensor.is_const && @context[:_cache][cache_key]
+        @context[cache_key] = if tensor.value.is_a?(Tensor)
+                                _run(tensor.value, child_context)
+                              else
+                                wrap_opencl(tensor, name: tensor.name)
+                              end
+        @context[:_cache][cache_key] = @context[cache_key] if tensor.is_const
+        @context[cache_key]
+      end
+
+      private
+
+      def assign_var(tensor, b, child_context)
+        assign = tensor.inputs[0] || tensor
+        buffer = complete_eval(b, child_context)
+
+        if assign.buffer
+          # buffer = type_cast(buffer, assign.data_type, name: "#{tensor.name}/cast_#{tensor.name}_#{tensor.data_type}")
+          event_wait_list = build_event_wait_list([buffer, assign.buffer])
+          assign.buffer.op = if assign.buffer.cl_buffer != buffer.cl_buffer
+                               _opencl_queue.enqueue_copy_buffer(buffer.cl_buffer, assign.buffer.cl_buffer, event_wait_list: event_wait_list)
+                             else
+                               buffer.op
+                             end
+        else
+          value = read_final_result(buffer)
+          assign.buffer = convert_to_opencl(value, buffer.shape, data_type: tensor.data_type, name: assign.name)
+          assign.value = value
+        end
+        assign.buffer.dirty = true
+        assign.buffer
+      end
+
+      def execute_2_operand_func(op_name, tensor, input_a, input_b, child_context, prog_name = nil)
+        a = _run(input_a, child_context)
+        b = _run(input_b, child_context)
+        a, b = auto_type_cast(a, b, name: "#{tensor.name}/cast_#{a.name}_#{b.data_type}")
+        dtype = tensor.data_type
+        result_shape = TensorShape.infer_shape(a.shape, b.shape)
+        return _create_result_buffer(dtype, [0], "out_#{tensor.name}") if result_shape == [0]
+        output_buffer = _create_result_buffer(tensor.data_type, result_shape, "out_#{tensor.name}")
+        a, b, prog, switch_operands = select_program(a, b, op_name)
+        m, n = result_shape
+        work_group = [m || 1, n || 1]
+        cl_m = OpenCL::Int1.new(m || 1)
+        cl_n = OpenCL::Int1.new(n || 1)
+        cl_switch = OpenCL::Int1.new(switch_operands) # no need to switch for addition
+
+        event_wait_list = build_event_wait_list([a, b]) # add dependency wait list
+
+        method_call = :"#{prog}_#{a.data_type}_#{b.data_type}"
+        prog_name ||= op_name
+        event = if prog == "#{op_name}_b"
+                  cl_m_b, cl_n_b = if b.shape.size == 2
+                                     [OpenCL::Int1.new(b.shape[0]), OpenCL::Int1.new(b.shape[1])]
+                                   elsif b.shape.size == 1
+                                     [OpenCL::Int1.new(1), OpenCL::Int1.new(b.shape[0])]
+                                   else
+                                     raise "rank > 2 not supported!"
+                                   end
+                  _cl_program(prog_name, a: a.data_type, b: b.data_type, dtype: dtype).
+                    send(method_call, _opencl_queue, work_group, cl_m, cl_n, cl_m_b, cl_n_b,
+                         cl_switch, a.cl_buffer, b.cl_buffer, output_buffer.cl_buffer, event_wait_list: event_wait_list)
+                else
+                  _cl_program(prog_name, a: a.data_type, b: b.data_type, dtype: dtype).
+                    send(method_call, _opencl_queue, work_group, cl_m, cl_n, cl_switch,
+                         a.cl_buffer, b.cl_buffer, output_buffer.cl_buffer, event_wait_list: event_wait_list)
+                end
+
+        output_buffer.op = event
+        output_buffer
+      end
+
+      def execute_cond_func(op_name, tensor, pred, input_a, input_b, child_context)
+        p = _run(pred, child_context)
+        a = _run(input_a, child_context)
+        b = _run(input_b, child_context)
+
+        a, b = auto_type_cast(a, b, name: "#{tensor.name}/cast_#{a.name}_#{b.data_type}")
+        dtype = tensor.data_type
+
+        output_buffer = _create_result_buffer(tensor.data_type, p.shape, tensor.name)
+
+        m, n = p.shape
+        work_group = [m || 1, n || 1]
+        cl_m = OpenCL::Int1.new(m || 1)
+        cl_n = OpenCL::Int1.new(n || 1)
+
+        event_wait_list = build_event_wait_list([a, b, p]) # add dependency wait list
+        output_buffer.op = _cl_program(op_name.to_s, dtype: dtype).send(:"#{op_name}_#{dtype}", _opencl_queue, work_group, cl_m, cl_n, p.cl_buffer, a.cl_buffer, b.cl_buffer, output_buffer.cl_buffer, event_wait_list: event_wait_list)
+        output_buffer
+      end
+
+      def execute_func(op_name, tensor, a, child_context)
+        a = _run(a, child_context)
+        event_wait_list = build_event_wait_list([a])
+        dtype = tensor.data_type
+        output_buffer = _create_result_buffer(tensor.data_type, a.shape, tensor.name)
+
+        m, n = a.shape
+        work_group = [m || 1, n || 1]
+        cl_m = OpenCL::Int1.new(m || 1)
+        cl_n = OpenCL::Int1.new(n || 1)
+
+        event = _cl_program(op_name.to_s, dtype: dtype).send(:"#{op_name}_#{dtype}", _opencl_queue, work_group, cl_m, cl_n, a.cl_buffer, output_buffer.cl_buffer, event_wait_list: event_wait_list)
+        output_buffer.op = event
+        output_buffer
+      end
+
+      def auto_type_cast(a, b, name: nil)
+        return [a, b] if a.data_type == b.data_type
+        m, n = b.shape
+        work_group = [m || 1, n || 1]
+        event_wait_list = build_event_wait_list([b])
+        buffer = _create_result_buffer(b.data_type, b.shape, name)
+
+        cl_m = OpenCL::Int1.new(m || 1)
+        cl_n = OpenCL::Int1.new(n || 1)
+
+        buffer.op = _cl_program("cast", source_dt: a.data_type, target_dt: b.data_type).cast(_opencl_queue, work_group, cl_m, cl_n, b.cl_buffer, buffer.cl_buffer, event_wait_list: event_wait_list)
+        [a, buffer]
+      end
+
+      def type_cast(source, data_type, name: nil)
+        return source if source.data_type == data_type
+        m, n = source.shape
+        work_group = [m || 1, n || 1]
+        event_wait_list = [source.op].compact
+        buffer = _create_result_buffer(data_type, source.shape, name)
+
+        cl_m = OpenCL::Int1.new(m || 1)
+        cl_n = OpenCL::Int1.new(n || 1)
+
+        buffer.op = _cl_program("cast", source_dt: source.data_type, target_dt: data_type).cast(_opencl_queue, work_group, cl_m, cl_n, source.cl_buffer, buffer.cl_buffer, event_wait_list: event_wait_list)
+        buffer
+      end
+
+      def wrap_opencl(tensor, data_type: nil, name: nil)
+        value, shape = if tensor.is_a?(Tensor)
+                         [tensor.value, tensor.shape.shape]
+                       else
+                         [tensor, shape_eval(tensor)]
+                       end
+
+        convert_to_opencl(value, shape, data_type: data_type || tensor.data_type, name: name)
+      end
+
+      def get_cached_buffer(name, shape)
+        cache_key = "_cl_object_#{name}:#{shape.join('_')}:#{object_id}"
+        @context[:_cache][cache_key]
+      end
+
+      def convert_to_opencl(value, shape, data_type: nil, name: nil)
+        value = [value] if !value.is_a?(Array) && !value.is_a?(NArray)
+
+        cache_key = "_cl_object_#{name}:#{shape.join('_')}:#{object_id}"
+        cl_object = if name && @context[:_cache][cache_key]
+                      @context[:_cache][cache_key]
+                    else
+                      narray_size = shape.reduce(:*) || 1
+
+                      buffer = if value.is_a?(NArray)
+                                 value
+                               else
+                                 allocate_narray_for_type(data_type, narray_size)
+                               end
+
+                      return nil if buffer.nil?
+
+                      cl_buffer_size = shape.empty? ? 1 : shape.reduce(:*)
+
+                      cl_buffer = unless value.flatten.empty?
+                                    cl_buffer_size = 1 if cl_buffer_size.zero?
+                                    _opencl_context.create_buffer(cl_buffer_size * buffer.element_size)
+                                  end
+
+                      @context[:_cache][cache_key] = OpenCLBuffer.new(name: name, data_type: data_type, shape: shape, buffer: buffer, cl_buffer: cl_buffer)
+                    end
+
+        if value.is_a?(Array)
+          value.flatten.each_with_index do |element, index|
+            cl_object.buffer[index] = if element.is_a?(Tensor)
+                                        read_final_result(complete_eval(element, {}))
+                                      elsif data_type == :boolean
+                                        element ? 1 : 0
+                                      else
+                                        Tensor.cast_dtype(element, data_type)
+                                      end
+          end
+        elsif value.is_a?(NArray)
+          cl_object.buffer = value
+        elsif data_type == :boolean
+          cl_object.buffer[0] = element ? 1 : 0
+        else
+          cl_object.buffer[0] = Tensor.cast_dtype(value, data_type)
+        end
+
+        write_op = _opencl_queue.enqueue_write_buffer(cl_object.cl_buffer, cl_object.buffer) if cl_object.cl_buffer && !value.nil? && (!value.is_a?(Array) || !value.empty?)
+        cl_object.op = write_op
+        cl_object
+      end
+
+      def allocate_narray_for_type(data_type, narray_size)
+        case data_type
+        when :float, :float32
+          NArray.sfloat(narray_size)
+        when :float64
+          NArray.float(narray_size)
+        when :int, :int32, :int64
+          NArray.int(narray_size)
+        when :int16
+          NArray.sint(narray_size)
+        when :boolean
+          NArray.sint(narray_size)
+        when :unknown
+          nil
+        else
+          raise "unsupported type #{data_type}"
+        end
+      end
+
+      def _create_result_buffer(data_type, shape, name)
+        return OpenCLBuffer.new(name: name, data_type: data_type, shape: [0], buffer: nil, cl_buffer: nil) if shape == [0]
+        @context[:_cache][:_cl_buffers]["_result_#{name}_#{shape.join('_')}:#{object_id}"] ||= begin
+          size = shape.empty? || shape == [0] ? 1 : shape.reduce(:*)
+          buffer =  allocate_narray_for_type(data_type, size)
+          cl_buffer = _opencl_context.create_buffer(buffer.size * buffer.element_size)
+          OpenCLBuffer.new(data_type: data_type, shape: shape, buffer: buffer, cl_buffer: cl_buffer, name: name)
+        end
+      end
+
+      def get_op_with_axis(a, target_axis, current_axis, output_type, op = ->(t, u) { t > u })
+        if target_axis == current_axis
+          if a[0].is_a?(Array)
+            (0...a[0].size).each.collect do |column_index|
+              max = nil
+              max_index = 0
+              a.each_with_index do |row, row_index|
+                if max.nil? || op.call(row[column_index], max)
+                  max = row[column_index]
+                  max_index = row_index
+                end
+              end
+
+              Tensor.cast_dtype(max_index, output_type)
+            end
+          else
+            max = nil
+            max_index = 0
+            a.each_with_index do |x, index|
+              if max.nil? || op.call(x, max)
+                max = x
+                max_index = index
+              end
+            end
+            Tensor.cast_dtype(max_index, output_type)
+          end
+        else
+          a.collect do |row|
+            get_op_with_axis(row, target_axis, current_axis + 1, output_type, op)
+          end
+        end
+      end
+
+      def _reduced_shape(input_shape, axes)
+        return [] if axes.nil? # reduce to scalar
+        axes = [axes] unless axes.is_a?(Array)
+        return input_shape if axes.empty?
+
+        axes.each do |dimen|
+          input_shape[dimen] = 1
+        end
+        input_shape
+      end
+
+      def reduction(child_context, tensor, a, b, func)
+        input = complete_eval(a, child_context)
+        axis = b.is_a?(Tensor) ? read_final_result(complete_eval(b, child_context)) : b
+        if axis.nil?
+          red = input.buffer.send(func)
+          convert_to_opencl(red, [], data_type: tensor.data_type, name: tensor.name)
+        else
+          return input if input.shape.empty?
+          value = input.buffer.reshape(*input.shape.reverse)
+          rank = input.shape.size - 1
+
+          if axis.is_a?(Array)
+            axis.map { |x| rank - x.abs }.sort.reverse_each do |x|
+              value = value.send(func, x.to_i)
+            end
+          else
+            value = value.send(func, rank - axis.abs)
+          end
+
+          new_shape = if value.is_a?(NArray)
+                        value.shape.reverse
+                      else
+                        value = [value]
+                        []
+                      end
+
+          new_shape = _reduced_shape(input.shape.dup, axis) if tensor.options[:keepdims]
+
+          convert_to_opencl(value.flatten, new_shape, data_type: tensor.data_type, name: tensor.name)
+        end
+      end
+
+      # selects variants of cl programs depending on input
+      def select_program(input_a, input_b, op)
+        return [input_a, input_b, op.to_s, 0] if input_a.shape == input_b.shape
+
+        return [input_b, input_a, "#{op}_c", 1] if input_a.shape.empty? || input_a.shape.reduce(:*) == 1 # A is scalar?
+        return [input_a, input_b, "#{op}_c", 0] if input_b.shape.empty? || input_a.shape.reduce(:*) == 1 # B is scalar?
+
+        return [input_b, input_a, "#{op}_b", 1] if input_a.shape.size < input_b.shape.size
+
+        if input_a.shape.size == input_b.shape.size
+          input_a.shape.zip(input_b.shape).each do |s1, s2|
+            return [input_b, input_a, "#{op}_b", 1] if s1 < s2
+          end
+        end
+
+        [input_a, input_b, "#{op}_b", 0]
+      end
+
+      def _rank_from_shape(shape)
+        shape.is_a?(Array) ? shape.size : 0
+      end
+
+      def build_event_wait_list(inputs)
+        inputs.compact.map(&:op).flatten
+      end
+
+      def resolve_placeholder(placeholder, _execution_context = {})
+        return nil if placeholder.nil?
+
+        var = if placeholder.is_a?(Placeholder)
+                @context[placeholder.name.to_sym].tap do |c|
+                  raise "missing placeholder #{placeholder.name}" if c.nil?
+                end
+              else
+                placeholder
+              end
+
+        return convert_to_opencl(var, shape_eval(var), data_type: placeholder.data_type, name: placeholder.name) unless var.is_a?(Tensor)
+        Tensor.cast_dtype(var, placeholder.data_type)
+      end
+
+      def all_true?(arr)
+        if arr.is_a?(Array) || arr.is_a?(NArray)
+          arr.each do |a|
+            return false unless all_true?(a)
+          end
+          return true
+        end
+
+        arr != 0
+      end
+    end
+  end
+end
+
+TensorStream::Evaluator.register_evaluator(TensorStream::Evaluator::OpenclEvaluator, 'opencl', 1)