ruby-cntk 0.1.0.pre1 → 0.1.0.pre2

data/ext/cntk/extconf.rb

@@ -1,21 +1,13 @@
-require "rbconfig"
-
-#RbConfig::MAKEFILE_CONFIG["CXX"] = "clang++-3.8"
-
 require 'mkmf'
 
-
-
-
 have_library("c++") or have_library("stdc++")
 
-# have_library("cntklibrary-2.0", nil, nil, " -L /cntk/build/cpu/release/lib/ ")
 dir_config("cntklibrary-2.0")
 have_library("cntklibrary-2.0")
 
 # rake2.3 compile -- --with-cntklibrary-2.0-lib=/cntk/build/cpu/release/lib/ --with-cntklibrary-2.0-include=/cntk/Source/CNTKv2LibraryDll/API/
 # rake2.3 compile -- --with-cntklibrary-2.0-lib=/cntk/cntk/lib/ --with-cntklibrary-2.0-include=/cntk/Include/
-$CXXFLAGS = ($CXXFLAGS || "") + " -std=c++11 -O2 -DSWIG " 
-# $LDFLAGS = ($LDFLAGS || "") + ""
+
+$CXXFLAGS = ($CXXFLAGS || "") + " -std=c++11 -O2 -DSWIG -Wl,--whole-archive " 
 
 create_makefile('cntk/CNTK')

data/lib/cntk/axis.rb

@@ -0,0 +1,18 @@
+module CNTK
+class Axis
+class << self
+  def from_num(n)
+    case n
+    when nil
+      all_static_axes
+    when Numeric
+      new(-n-1)
+    when Axis
+      new(-1-n.static_axis_index)
+    else
+      raise ArgumentError
+    end
+  end
+end
+end
+end

data/lib/cntk/dictionary.rb

@@ -0,0 +1,62 @@
+module CNTK
+
+  class DictionaryValue
+    
+    def self.create(val)
+      case val
+      when Hash
+        new Dictionary.create(val)
+      when Array
+        v = StdVectorDictionaryValue.new
+        val.each_with_index{|e, idx|
+          v[idx] = create(e)
+        }
+        new( v )
+      else
+        new val
+      end
+    end
+
+    def value
+      case value_type
+      when Type_Bool
+        value_bool__
+      when Type_Int
+        value_int__
+      when Type_SizeT
+        value_size_t__
+      when Type_Float
+        value_float__
+      when Type_Double
+        value_double__
+      when Type_String
+        value_string__
+      when Type_NDShape
+        value_ndshape__
+      when Type_Axis
+        value_axis__
+      when Type_Vector
+        value_vec_dict_value__
+      when Type_Dictionary
+        value_dict__
+      when Type_NDArrayView
+        value_ndarrayview__
+      else
+        raise "unknown type"
+      end
+    end
+
+  end
+
+  class Dictionary
+    def self.create(h)
+      dict = new()
+      h.each_pair{|k, v|
+        k = k.to_s if k.is_a?(Symbol)
+        dict[k] = DictionaryValue.create(v)
+      }
+      return dict
+    end
+  end
+
+end

data/lib/cntk/function.rb

@@ -1,61 +1,113 @@
 module CNTK
+
   class Function
 
-    def call(args)
-      if args.outputs.length == 1
-        return replace_placeholders({placeholders[0] => args.output})
+    def dot(other)
+      output.dot(other)
+    end
+
+    def -@
+      - output
+    end
+
+    def +(other)
+      output + other
+    end
+
+    def -(other)
+      output - other
+    end
+
+    def *(other)
+      output * other
+    end
+
+    def /(other)
+      output / other
+    end
+
+    # FIXME
+    def coerce(other)
+      if other.is_a?(Numeric)
+        [Constant::scalar(output.get_data_type, other), self]
+      else
+        
+      end
+    end
+
+    def call(func)
+      if func.respond_to?(:output)
+        val = func.output
       else
-        raise "not implemented"
+        val = func
       end
+      if placeholders().length == 1
+        replace_placeholders({placeholders[0] => val})
+      else
+        raise "the number of placeholders is not 1."
+      end
+    end
+
+    # forward function composition self(func(...))
+    def >>(func)
+      func.call(self)
+    end
+
+    def <<(func)
+      call(func)
+    end
+
+    def forward(argsmap, outputs = [], keep_for_backward: [], device: DeviceDescriptor.use_default_device(), remove_dynamic_axes: true)
+      input = convert_to_value(argsmap)
+      out = StdUMapVariableValue.new()
+      outputs.each{|out_var|
+        # By setting nullptr, Forward function implemented in C++ will allocate Value object with required storage.
+        out.__set_nullptr__(out_var)
+      }
+      b = __forward__(input, out, device, keep_for_backward)
+      # FIXME. we will remove this line.
+      out = remove_dynamic_axes(out) if remove_dynamic_axes
+      return [b, out]
     end
 
-    def forward(*args)
-      if args.length > 1
-        return __forward__(*args)
-      elsif args.length == 1
-        input = convert_to_value(args[0])
-        out = StdUMapVariableValue.new()
-        outputs().each{|o|
-          v = NDArrayView.new(CNTK::DataType_Double, 
-                              required_output_shape(o),
-                              required_output_buf(o),
-                              CNTK::DeviceDescriptor.default_device(),
-                              true)
-          out[o] = Value.new(v)
-        }
-        b = __forward__(input, out)
-        out = remove_dynamic_axes(out)
-        return [out, b]
+    def eval(argsmap=nil, device: DeviceDescriptor.use_default_device(), remove_dynamic_axes: true)
+      argsmap = {} if argsmap == nil
+      _, outmap = forward(argsmap, outputs(), device: device, remove_dynamic_axes: remove_dynamic_axes)
+      if outmap.size > 1
+        outmap
+      else
+        outmap.values[0]
       end
     end
 
+    def backward(state, root_gradients, variables, remove_dynamic_axes: true)
+      root_gradients = convert_to_value(root_gradients)
+      out = StdUMapVariableValue.new()
+      variables.each{|var|
+        out.__set_nullptr__(var)
+      }
+      __backward__(state, root_gradients, out)
+      out = remove_dynamic_axes(out)
+    end
+
     def convert_to_value(h)
-      input = {}
+      ret = {}
       h.each_pair{|k,val|
         if val.respond_to?(:row_major?)
-          input[k] = Value.create(val)
+          ret[k] = Value.new(NDArrayView.create(val))
         else
-          input[k] = val
+          ret[k] = val
         end
       }
-      return input
-    end
-
-    #FIXME
-    # we must add dynamic axes?
-    def required_output_shape(ov)
-      ov.shape().to_a + [1,1]
-    end
-
-    def required_output_buf(ov)
-      [1.0] * ov.shape.total_size
+      return ret
     end
 
     def remove_dynamic_axes(out)
       out1 = {}
       out.each{|o,ov|
-        if ov.shape.rank == o.shape.rank + 2 and ov.shape.to_a[-2..-1] == [1,1]
-          out1[o] = ov.reshape( ov.shape.to_a[0..-3] )
+        sz = o.dynamic_axes.size
+        if sz > 0 and sz < ov.shape.rank and ov.shape.to_a[0..1] == [1,1]
+          out1[o] = ov.reshape( ov.shape.to_a[sz..-1] )
         else
           out1[o] = ov
         end

data/lib/cntk/initializer.rb

@@ -0,0 +1,67 @@
+module CNTK
+module Initializer
+class << self
+  
+  def constant
+    CNTK.__constant_initializer__
+  end
+
+  def uniform(scale, seed = CNTK.SentinelValueForAutoSelectRandomSeed)
+      CNTK.__uniform_initializer__(scale, seed)
+  end
+
+  def normal(scale,
+             output_rank: CNTK.SentinelValueForInferParamInitRank,
+             filter_rank: CNTK.SentinelValueForInferParamInitRank,
+             seed:        CNTK.SentinelValueForAutoSelectRandomSeed)
+    CNTK.__normal_initializer__(scale, output_rank, filter_rank, seed)
+  end
+
+  def glorot_uniform(scale = CNTK.DefaultParamInitScale,
+                     output_rank: CNTK.SentinelValueForInferParamInitRank,
+                     filter_rank: CNTK.SentinelValueForInferParamInitRank,
+                     seed:        CNTK.SentinelValueForAutoSelectRandomSeed)
+    CNTK.__glorot_uniform_initializer__(scale, output_rank, filter_rank, seed)
+  end
+
+  def glorot_normal(scale = CNTK.DefaultParamInitScale,
+                    output_rank: CNTK.SentinelValueForInferParamInitRank,
+                    filter_rank: CNTK.SentinelValueForInferParamInitRank,
+                    seed:        CNTK.SentinelValueForAutoSelectRandomSeed)
+    CNTK.__glorot_normal_initializer__(scale, output_rank, filter_rank, seed)
+  end
+
+  def xavier(scale,
+             output_rank: CNTK.SentinelValueForInferParamInitRank,
+             filter_rank: CNTK.SentinelValueForInferParamInitRank,
+             seed:        CNTK.SentinelValueForAutoSelectRandomSeed)
+    CNTK.__xavier_initializer__(scale, output_rank, filter_rank, seed)
+  end
+
+  def he_uniform(scale = CNTK.DefaultParamInitScale,
+                 output_rank: CNTK.SentinelValueForInferParamInitRank,
+                 filter_rank: CNTK.SentinelValueForInferParamInitRank,
+                 seed:        CNTK.SentinelValueForAutoSelectRandomSeed)
+    CNTK.__he_uniform_initializer__(scale, output_rank, filter_rank, seed)
+  end
+
+  def he_normal(scale = CNTK.DefaultParamInitScale,
+                output_rank: CNTK.SentinelValueForInferParamInitRank,
+                filter_rank: CNTK.SentinelValueForInferParamInitRank,
+                seed:        CNTK.SentinelValueForAutoSelectRandomSeed)
+    CNTK.__he_normal_initializer__(scale, output_rank, filter_rank, seed)
+  end
+
+  def bilinear(kernel_width, kernel_height)
+    CNTK.__bilinear_initializer__(kernel_width, kernel_height)
+  end
+
+  def initializer_with_rank(initializer,
+                            output_rank: CNTK.SentinelValueForInferParamInitRank,
+                            filter_rank: CNTK.SentinelValueForInferParamInitRank)
+    CNTK.__random_initializer_with_rank__(initializer, output_rank, filter_rank)
+  end
+
+end # class << self
+end # module Initializer
+end # module CNT

data/lib/cntk/inspect.rb

@@ -0,0 +1,57 @@
+module CNTK
+  module InspectUtil
+    def inspect_methods_p(mthds)
+      mthds.map{|mth| "#{mth}=" + send(mth).inspect }.join(", ")
+    end
+
+    def inspect_methods(mthds)
+      s = inspect_methods_p(mthds)
+      "#<#{self.class} #{s}>"
+    end
+  end
+
+  class StdUMapStreamInfoMinibatchData
+    include InspectUtil
+    def inspect
+      s = "{" + map{|k, v| k.inspect + " => " + v.inspect }.join(", ") + "}"
+      "#<#{self.class}: #{s}>"
+    end
+  end
+
+  class Axis
+    include InspectUtil
+    def inspect
+      inspect_methods([:name, :is_dynamic_axis])
+    end
+  end
+
+  class NDShape
+    def inspect
+      to_a.inspect
+    end
+  end
+
+  class Value
+    include InspectUtil
+    def inspect
+      inspect_methods([:shape])
+    end
+  end
+
+  class StreamInformation
+    include InspectUtil
+
+    def inspect
+      inspect_methods([:name, :id])
+    end
+  end
+
+  class MinibatchData
+    include InspectUtil
+
+    def inspect
+      inspect_methods([:data, :number_of_samples])
+    end
+  end
+
+end

data/lib/cntk/io.rb

@@ -0,0 +1,50 @@
+module CNTK
+
+  def self.create_composite_minibatch_source(dict)
+    if dict.respond_to?(:to_hash)
+      h = {}
+      dict.to_hash.each_pair{|k, v|
+        k = k.to_s if k.is_a?(Symbol)
+        h[k] = v
+      }
+      des = h["deserializers"]
+      unless des.respond_to?(:to_ary)
+        h["deserializers"] = [des]
+      end
+      dict = Dictionary.create(h)
+    end
+    CNTK.__create_composite_minibatch_source__(dict)
+  end
+
+  class MinibatchSource
+
+    # @param minibatch_size_in_samples [Integer]
+    # @param device                    [DeviceDescriptor]
+    # @param num_data_partitions       [Integer]
+    # @param partition_index           [Integer]
+    # @return [MinibatchData]
+    def next_minibatch(minibatch_size_in_samples, device: DeviceDescriptor.use_default_device,
+                       num_data_partitions: 1, partition_index: 0)
+      get_next_minibatch(0, minibatch_size_in_samples, num_data_partitions, partition_index, device)
+    end
+
+  end
+
+  # std::unordered_map<StreamInfo, MinibatchData>
+  class MinibatchTable 
+    alias __get__ :[]
+    def [](key)
+      if key.respond_to?(:to_str)
+        key = key.to_str
+        a = self.keys.find_all{|k| k.name == key }
+        if a.size > 1
+          raise "The number of input data having the name is not 1."
+        end
+        __get__(a[0])
+      else
+        __get__(key)
+      end
+    end
+  end
+
+end

data/lib/cntk/layers.rb

@@ -0,0 +1,17 @@
+module CNTK
+module Layers
+
+  module_function
+
+  # @return [Function]
+  def dense(output_shape, init: Initializer.glorot_uniform,
+            input_shape: [CNTK::NDShape::InferredDimension],
+            use_bias: true, init_bias: 0, name: "")
+    _W = Ops.parameter(shape: input_shape + output_shape, init: init,      name: "W")
+     b = Ops.parameter(shape:               output_shape, init: init_bias, name: "b")
+     x = Ops.placeholder_variable(name: "x")
+     Ops.times(x, _W, output_rank: output_shape.size, infer_input_rank_to_map: 0) + b
+  end
+
+end
+end

data/lib/cntk/learner.rb

@@ -0,0 +1,197 @@
+module CNTK
+
+class Learner
+  LearningRateSchedule = MomentumSchedule
+  MinibatchSizeSchedule = TrainingParameterPerSampleSchedule
+
+  # TrainingParameterPerSampleSchedule    == MinibatchSizeSchedule 
+  # TrainingParameterPerMinibatchSchedule 
+  #
+class << self
+
+private
+
+  def create_opt(l1_weight, l2_weight, ga, threshold, truncation)
+    opt = AdditionalLearningOptions.new
+    opt.l1_regularization_weight = l1_weight
+      opt.l2_regularization_weight = l2_weight
+    opt.gaussian_noise_injection_std_dev = ga
+    opt.gradient_clipping_threshold_per_sample = threshold
+    opt.gradient_clipping_with_truncation = truncation
+
+    return opt
+  end
+
+public
+
+  # @param schedule   [Numeric, Array<Numeric>]
+  # @param unit       [:sample, :minibatch]
+  # @param epoch_size [Numeric]
+  # @return [TrainingParameterPerSampleSchedule, TrainingParameterPerMinibatchSchedule]
+  def training_parameter_schedule(schedule, unit, epoch_size = nil)
+    case unit
+    when :sample
+      klass = TrainingParameterPerSampleSchedule
+    when :minibatch
+      klass = TrainingParameterPerMinibatchSchedule
+    else
+      raise "unknown unit"
+    end
+
+    if schedule.is_a?(Numeric) 
+      unless epoch_size.nil?
+        raise "epoch_size can't be given when schedule is Numeric."
+      else
+        klass.new(schedule)
+      end
+    else
+      if epoch_size.nil?
+        klass.new(schedule)
+      else
+        klass.new(schedule, epoch_size)
+      end
+    end
+
+  end
+
+  # @param schedule   [Numeric, Array<Numeric>]
+  # @param unit       [:sample, :minibatch]
+  # @param epoch_size [Numeric]
+  # @return [TrainingParameterPerSampleSchedule, TrainingParameterPerMinibatchSchedule]
+  def momentum_schedule(schedule, unit = :minibatch, epoch_size = nil)
+    training_parameter_schedule(schedule, unit, epoch_size)
+  end
+
+  # @param schedule   [Numeric, Array<Numeric>]
+  # @param epoch_size [Numeric]
+  # @return [MomentumAsTimeConstantSchedule]
+  def momentum_as_time_constant_schedule(schedule, epoch_size)
+    klass = MomentumAsTimeConstantSchedule
+    if schedule.is_a?(Numeric) 
+      if epoch_size.nil?
+        raise "epoch_size can't be given when schedule is Numeric."
+      else
+        klass.new(schedule)
+      end
+    else
+      if epoch_size.nil?
+        klass.new(schedule)
+      else
+        klass.new(schedule, epoch_size)
+      end
+    end
+  end
+  
+  # @param parameters [Array<Parameter>]
+  # @param lr         [TrainingParameterPerSampleSchedule, TrainingParameterPerMinibatchSchedule]
+  # @option opt [Float] :l1_weight
+  # @option opt [Float] :l2_weight
+  # @option opt [Float] :std_dev
+  # @option opt [Float] :threshold
+  # @option opt [Boolean] :truncation
+  # @return [Learner]
+  def sgd(parameters, lr, l1_weight: 0.0, l2_weight: 0.0,
+                       std_dev: 0.0, threshold: Float::INFINITY, truncation: true)
+    ga = training_parameter_schedule(std_dev, :minibatch)
+    opt = create_opt(l1_weight, l2_weight, ga, threshold, truncation)
+    CNTK.__sgdlearner__(parameters, lr, opt)
+  end
+
+  # @param parameters [Array<Parameter>]
+  # @param lr         [TrainingParameterPerSampleSchedule, TrainingParameterPerMinibatchSchedule]
+  # @param momentum   [MomentumSchedule]
+  # @param unit_gain  [Boolean]
+  # @option opt [Float] :l1_weight
+  # @option opt [Float] :l2_weight
+  # @option opt [Float] :std_dev
+  # @option opt [Boolean] :truncation
+  # @return [Learner]
+  def momentum_sgd(parameters, lr, momentum, unit_gain: CNTK.default_unit_gain_value(),
+                                l1_weight: 0.0, l2_weight: 0.0,
+                                std_dev: 0.0, threshold: Float::INFINITY, truncation: true)
+    ga = training_parameter_schedule(std_dev, :minibatch)
+    opt = create_opt(l1_weight, l2_weight, ga, threshold, truncation)
+    CNTK.__momentum_sgd_learner__(parameters, lr, momentum, unit_gain, opt)
+  end
+
+  # @param parameters [Array<Parameter>]
+  # @param lr         [TrainingParameterPerSampleSchedule, TrainingParameterPerMinibatchSchedule]
+  # @param momentum   [MomentumSchedule]
+  # @param unit_gain  [Boolean]
+  # @option opt [Float] :l1_weight
+  # @option opt [Float] :l2_weight
+  # @option opt [Float] :std_dev
+  # @option opt [Boolean] :truncation
+  # @return [Learner]
+  def nesterov(parameters, lr, momentum, unit_gain: CNTK.default_unit_gain_value(),
+               l1_weight: 0.0, l2_weight: 0.0,
+               std_dev: 0.0, threshold: Float::INFINITY, truncation: true)
+    ga = training_parameter_schedule(std_dev, :minibatch)
+    opt = create_opt(l1_weight, l2_weight, ga, threshold, truncation)
+    CNTK.__nesterov_learner__(parameters, lr, momentum, unit_gain, opt)
+  end
+
+  # @param parameters [Array<Parameter>]
+  # @param lr         [TrainingParameterPerSampleSchedule, TrainingParameterPerMinibatchSchedule]
+  # @param momentum   [MomentumSchedule]
+  # @param unit_gain  [Boolean]
+  # @option opt [Float] :l1_weight
+  # @option opt [Float] :l2_weight
+  # @option opt [Float] :std_dev
+  # @option opt [Boolean] :truncation
+  # @return [Learner]
+  def adagrad(parameters, lr, multiplier: true, unit_gain: CNTK.default_unit_gain_value(),
+               l1_weight: 0.0, l2_weight: 0.0,
+               std_dev: 0.0, threshold: Float::INFINITY, truncation: true)
+    ga = training_parameter_schedule(std_dev, :minibatch)
+    opt = create_opt(l1_weight, l2_weight, ga, threshold, truncation)
+    CNTK.__ada_grad_learner__(parameters, lr, multiplier, unit_gain, opt)
+  end
+
+  # @param parameters [Array<Parameter>]
+  # @param lr         [TrainingParameterPerSampleSchedule, TrainingParameterPerMinibatchSchedule]
+  # @param momentum   [MomentumSchedule]
+  # @param unit_gain  [Boolean]
+  # @param variance_momentum [MomentumAsTimeConstantSchedule]
+  # @param low_memory [Boolean]
+  # @option opt [Float] :l1_weight
+  # @option opt [Float] :l2_weight
+  # @option opt [Float] :std_dev
+  # @option opt [Boolean] :truncation
+  # @return [Learner]
+  def adam_sgd(parameters, lr, momentum, unit_gain: CNTK.default_unit_gain_value(),
+               variance_momentum: momentum_as_time_constant_schedule(720000),
+               low_memory: true,
+               l1_weight: 0.0, l2_weight: 0.0,
+               std_dev: 0.0, threshold: Float::INFINITY, truncation: true)
+    ga = training_parameter_schedule(std_dev, :minibatch)
+    opt = create_opt(l1_weight, l2_weight, ga, threshold, truncation)
+    CNTK.__adam_learner__(parameters, lr, momentum, unit_gain, variance_momentum, low_memory, opt)
+  end
+
+  # @param parameters [Array<Parameter>]
+  # @param lr         [TrainingParameterPerSampleSchedule, TrainingParameterPerMinibatchSchedule]
+  # @param gamma [Float]
+  # @param inc   [Float]
+  # @param dec   [Float]
+  # @param max   [Float]
+  # @param min   [Float]
+  # @param multiplier [Boolean]
+  # @option opt [Float] :l1_weight
+  # @option opt [Float] :l2_weight
+  # @option opt [Float] :std_dev
+  # @option opt [Boolean] :truncation
+  # @return [Learner]
+  def rmsprop(parameters, lr, gamma, inc, dec, max, min,
+              multiplier: true, l1_weight: 0.0, l2_weight: 0.0,
+              std_dev: 0.0, threshold: Float::INFINITY, truncation: true)
+    ga = training_parameter_schedule(std_dev, :minibatch)
+    opt = create_opt(l1_weight, l2_weight, ga, threshold, truncation)
+    CNTK.__rmsprop_learner__(parameters, lr, gamma, inc, dec, max, min, multiplier, opt)
+  end
+
+end # class << self
+
+end # class Learner
+
+end # module CNTK

data/lib/cntk/ndarrayview.rb

@@ -2,25 +2,36 @@ module CNTK
   class NDArrayView
 
     def self.create(a)
-      if a.respond_to?(:shape) and a.respond_to?(:row_major?)
-        if a.row_major?
-          # NDArrayView is column-major.
-          # So we must transpose a.
-          ta = a.transpose
+      if a.respond_to?(:shape)
+        case a
+        when NDArrayView
+          return a
+        when Numo::DFloat
+          dtype = DataType_Double
+        when Numo::SFloat
+          dtype = DataType_Float
+        else
+          raise ArgumentError, "Numo::NArray or NDArrayView expected"
         end
-        return self.new(DataType_Double, a.shape, ta.flatten.to_a, 
+        return self.new(dtype, a.shape, a.flatten.to_a,
                         CNTK::DeviceDescriptor.default_device(), false)
       else
-        raise "not implemented"
+        raise ArgumentError, "not responds to :shape"
       end
     end
 
     def to_narray
-      ret = Numo::DFloat[*to_vec()]
-      # NDArrayView is column-major and NArray is row-major.
-      # So we must reverse shape and transpose it.
-      ret = ret.reshape(*shape().reverse)
-      return ret.transpose
+      case get_data_type
+      when DataType_Float
+        klass = Numo::SFloat
+      when DataType_Double
+        klass = Numo::DFloat
+      else
+        raise "unknown data type"
+      end
+      ret = klass[*to_vec()]
+      ret = ret.reshape(*shape().to_a)
+      return ret
     end
 
   end