tensorflow 0.1.1 → 0.1.2

data/lib/tensorflow/strings.rb

@@ -0,0 +1,100 @@
+module TensorFlow
+  module Strings
+    class << self
+      def as_string(input, precision: nil, scientific: nil, shortest: nil, width: nil, fill: nil)
+        RawOps.as_string(input: input, precision: precision, scientific: scientific, shortest: shortest, width: width, fill: fill)
+      end
+
+      # def bytes_split
+      # end
+
+      # def format
+      # end
+
+      def join(inputs, separator: "")
+        Utils.execute("StringJoin", inputs, separator: separator, N: inputs.size)
+      end
+
+      def length(input, unit: "BYTE")
+        RawOps.string_length(input: input, unit: unit)
+      end
+
+      def lower(input, encoding: "")
+        RawOps.string_lower(input: input, encoding: encoding)
+      end
+
+      # def ngrams
+      # end
+
+      def reduce_join(inputs, reduction_indices, keep_dims: nil, separator: nil)
+        RawOps.reduce_join(inputs: inputs, reduction_indices: reduction_indices, keep_dims: keep_dims, separator: separator)
+      end
+
+      def regex_full_match(input, pattern)
+        RawOps.regex_full_match(input: input, pattern: pattern)
+      end
+
+      def regex_replace(input, pattern, rewrite, replace_global: nil)
+        RawOps.regex_replace(input: input, pattern: pattern, rewrite: rewrite, replace_global: replace_global)
+      end
+
+      def split(split_dim, value, num_split: nil)
+        RawOps.split(split_dim: split_dim, value: value, num_split: num_split)
+      end
+
+      def strip(input)
+        RawOps.string_strip(input: input)
+      end
+
+      def substr(input, pos, len, unit: nil)
+        RawOps.substr(input: input, pos: pos, len: len, unit: unit)
+      end
+
+      # def to_hash_bucket
+      # end
+
+      # def to_hash_bucket_fast
+      # end
+
+      # def to_hash_bucket_strong
+      # end
+
+      def to_number(input, out_type: :float)
+        RawOps.string_to_number(string_tensor: input, out_type: out_type)
+      end
+
+      def unicode_decode(input, input_encoding: nil, errors: nil, replacement_char: nil, replace_control_characters: nil)
+        RawOps.unicode_decode(input: input, input_encoding: input_encoding, errors: errors, replacement_char: replacement_char, replace_control_characters: replace_control_characters)
+      end
+
+      def unicode_decode_with_offsets(input, input_encoding: nil, errors: nil, replacement_char: nil, replace_control_characters: nil)
+        RawOps.unicode_decode_with_offsets(input: input, input_encoding: input_encoding, errors: errors, replacement_char: replacement_char, replace_control_characters: replace_control_characters)
+      end
+
+      def unicode_encode(input_values, input_splits, errors: nil, output_encoding: nil, replacement_char: nil)
+        RawOps.unicode_encode(input_values: input_values, input_splits: input_splits, errors: errors, output_encoding: output_encoding, replacement_char: replacement_char)
+      end
+
+      def unicode_script(input)
+        RawOps.unicode_script(input: input)
+      end
+
+      # def unicode_split
+      # end
+
+      # def unicode_split_with_offsets
+      # end
+
+      def unicode_transcode(input, input_encoding: nil, output_encoding: nil, errors: nil, replacement_char: nil, replace_control_characters: nil)
+        RawOps.unicode_transcode(input: input, input_encoding: input_encoding, output_encoding: output_encoding, errors: errors, replacement_char: replacement_char, replace_control_characters: replace_control_characters)
+      end
+
+      # def unsorted_segment_join
+      # end
+
+      def upper(input, encoding: "")
+        RawOps.string_upper(input: input, encoding: encoding)
+      end
+    end
+  end
+end

data/lib/tensorflow/tensor.rb

@@ -6,7 +6,8 @@ module TensorFlow
       if pointer
         @pointer = pointer
       else
-        data = Array(value)
+        data = value
+        data = Array(data) unless data.is_a?(Array) || data.is_a?(Numo::NArray)
         shape ||= calculate_shape(value)
 
         if shape.size > 0
@@ -16,37 +17,49 @@ module TensorFlow
           dims_ptr = nil
         end
 
-        data = data.flatten
-
-        dtype ||= Utils.infer_type(data)
-        type = FFI::DataType[dtype]
-        case dtype
-        when :float, :double, :int32, :uint8, :int16, :int8, :int64, :uint16, :uint32, :uint64
-          data_ptr = ::FFI::MemoryPointer.new(dtype, data.size)
-          data_ptr.send("write_array_of_#{dtype}", data)
-        when :bfloat16
-          # https://en.wikipedia.org/wiki/Bfloat16_floating-point_format
-          data_ptr = ::FFI::MemoryPointer.new(:int8, data.size * 2)
-          data_ptr.write_bytes(data.map { |v| [v].pack("g")[0..1] }.join)
-        when :complex64
-          data_ptr = ::FFI::MemoryPointer.new(:float, data.size * 2)
-          data_ptr.write_array_of_float(data.flat_map { |v| [v.real, v.imaginary] })
-        when :complex128
-          data_ptr = ::FFI::MemoryPointer.new(:double, data.size * 2)
-          data_ptr.write_array_of_double(data.flat_map { |v| [v.real, v.imaginary] })
-        when :string
-          data_ptr = string_ptr(data)
-        when :bool
-          data_ptr = ::FFI::MemoryPointer.new(:int8, data.size)
-          data_ptr.write_array_of_int8(data.map { |v| v ? 1 : 0 })
+        if data.is_a?(Numo::NArray)
+          dtype ||= Utils.infer_type(data)
+          # TODO use Numo read pointer?
+          data_ptr = ::FFI::MemoryPointer.new(:uchar, data.byte_size)
+          data_ptr.write_bytes(data.to_string)
         else
-          raise "Unknown type: #{dtype}"
+          data = data.flatten
+          dtype ||= Utils.infer_type(data)
+          case dtype
+          when :float, :double, :int32, :uint8, :int16, :int8, :int64, :uint16, :uint32, :uint64
+            data_ptr = ::FFI::MemoryPointer.new(dtype, data.size)
+            data_ptr.send("write_array_of_#{dtype}", data)
+          when :bfloat16
+            # https://en.wikipedia.org/wiki/Bfloat16_floating-point_format
+            data_ptr = ::FFI::MemoryPointer.new(:int8, data.size * 2)
+            data_ptr.write_bytes(data.map { |v| [v].pack("g")[0..1] }.join)
+          when :complex64
+            data_ptr = ::FFI::MemoryPointer.new(:float, data.size * 2)
+            data_ptr.write_array_of_float(data.flat_map { |v| [v.real, v.imaginary] })
+          when :complex128
+            data_ptr = ::FFI::MemoryPointer.new(:double, data.size * 2)
+            data_ptr.write_array_of_double(data.flat_map { |v| [v.real, v.imaginary] })
+          when :string
+            data_ptr = string_ptr(data)
+          when :bool
+            data_ptr = ::FFI::MemoryPointer.new(:int8, data.size)
+            data_ptr.write_array_of_int8(data.map { |v| v ? 1 : 0 })
+          else
+            raise "Unknown type: #{dtype}"
+          end
         end
 
+        type = FFI::DataType[dtype]
+
         callback = ::FFI::Function.new(:void, [:pointer, :size_t, :pointer]) do |data, len, arg|
           # FFI handles deallocation
         end
 
+        # keep data pointer alive for duration of object
+        @data_ptr = data_ptr
+        @dims_ptr = dims_ptr
+        @callback = callback
+
         tensor = FFI.TF_NewTensor(type, dims_ptr, shape.size, data_ptr, data_ptr.size, callback, nil)
         @pointer = FFI.TFE_NewTensorHandle(tensor, @status)
         check_status @status
@@ -75,6 +88,10 @@ module TensorFlow
       Math.floormod(self, other)
     end
 
+    def -@
+      Math.negative(self)
+    end
+
     def value
       value =
         case dtype
@@ -92,10 +109,19 @@ module TensorFlow
           # https://github.com/tensorflow/tensorflow/blob/5453aee48858fd375172d7ae22fad1557e8557d6/tensorflow/c/tf_tensor.h#L57
           start_offset_size = element_count * 8
           offsets = data_pointer.read_array_of_uint64(element_count)
-          element_count.times.map { |i| (data_pointer + start_offset_size + offsets[i]).read_string }
+          byte_size = FFI.TF_TensorByteSize(tensor_pointer)
+          element_count.times.map do |i|
+            str_len = (offsets[i + 1] || (byte_size - start_offset_size)) - offsets[i]
+            str = (data_pointer + start_offset_size + offsets[i]).read_bytes(str_len)
+            dst = ::FFI::MemoryPointer.new(:char, str.bytesize + 100)
+            dst_len = ::FFI::MemoryPointer.new(:size_t)
+            FFI.TF_StringDecode(str, str.bytesize, dst, dst_len, @status)
+            check_status @status
+            dst.read_pointer.read_bytes(dst_len.read_int32)
+          end
         when :bool
           data_pointer.read_array_of_int8(element_count).map { |v| v == 1 }
-        when :resource
+        when :resource, :variant
           return data_pointer
         else
           raise "Unknown type: #{dtype}"
@@ -135,8 +161,14 @@ module TensorFlow
       @pointer
     end
 
+    def numo
+      klass = Utils::NUMO_TYPE_MAP[dtype]
+      raise "Unknown type: #{dtype}" unless klass
+      klass.cast(value)
+    end
+
     def inspect
-      inspection = %w(value shape dtype).map { |v| "#{v}: #{send(v).inspect}"}
+      inspection = %w(numo shape dtype).map { |v| "#{v}: #{send(v).inspect}"}
       "#<#{self.class} #{inspection.join(", ")}>"
     end
 
@@ -164,9 +196,13 @@ module TensorFlow
     end
 
     def data_pointer
+      FFI.TF_TensorData(tensor_pointer)
+    end
+
+    def tensor_pointer
       tensor = FFI.TFE_TensorHandleResolve(@pointer, @status)
       check_status @status
-      FFI.TF_TensorData(tensor)
+      tensor
     end
 
     def reshape(arr, dims)
@@ -179,6 +215,8 @@ module TensorFlow
     end
 
     def calculate_shape(value)
+      return value.shape if value.respond_to?(:shape)
+
       shape = []
       d = value
       while d.is_a?(Array)
@@ -199,7 +237,9 @@ module TensorFlow
       data_ptr = ::FFI::MemoryPointer.new(:char, start_offset_size + offsets.pop)
       data_ptr.write_array_of_uint64(offsets)
       data.zip(offsets) do |str, offset|
-        (data_ptr + start_offset_size + offset).write_string(str)
+        dst_len = FFI.TF_StringEncodedSize(str.bytesize)
+        FFI.TF_StringEncode(str, str.bytesize, data_ptr + start_offset_size + offset, dst_len, @status)
+        check_status @status
       end
       data_ptr
     end

data/lib/tensorflow/utils.rb

@@ -1,5 +1,18 @@
 module TensorFlow
   module Utils
+    NUMO_TYPE_MAP = {
+      int8: Numo::Int8,
+      int16: Numo::Int16,
+      int32: Numo::Int32,
+      int64: Numo::Int64,
+      uint8: Numo::UInt8,
+      uint16: Numo::UInt16,
+      uint32: Numo::UInt32,
+      uint64: Numo::UInt64,
+      float: Numo::SFloat,
+      double: Numo::DFloat
+    }
+
     class << self
       def check_status(status)
         if FFI.TF_GetCode(status) != 0
@@ -26,48 +39,104 @@ module TensorFlow
           type = FFI.TFE_OpGetAttrType(op, attr_name, is_list, status)
           check_status status
 
-          case FFI::AttrType[type]
-          when :string
-            FFI.TFE_OpSetAttrString(op, attr_name, attr_value, attr_value.bytesize)
-          # when :int
-          # when :float
-          # when :bool
-          when :type
-            FFI.TFE_OpSetAttrType(op, attr_name, attr_value)
-          when :shape
-            # TODO set value properly
-            FFI.TFE_OpSetAttrShape(op, attr_name, nil, 0, status)
-            check_status status
-          # when :tensor
-          # when :placeholder
-          # when :func
+          if is_list.read_int == 1
+            num_values = attr_value.size
+
+            case FFI::AttrType[type]
+            when :int
+              values = ::FFI::MemoryPointer.new(:int64, num_values)
+              values.write_array_of_int64(attr_value)
+              FFI.TFE_OpSetAttrIntList(op, attr_name, values, num_values)
+            when :float
+              values = ::FFI::MemoryPointer.new(:float, num_values)
+              values.write_array_of_float(attr_value)
+              FFI.TFE_OpSetAttrFloatList(op, attr_name, values, num_values)
+            when :shape
+              dims_ptrs =
+                attr_value.map do |shape|
+                  ptr = ::FFI::MemoryPointer.new(:int64, shape.size)
+                  ptr.write_array_of_int64(shape)
+                end
+              dims = ::FFI::MemoryPointer.new(:pointer, num_values)
+              dims.write_array_of_pointer(dims_ptrs)
+
+              num_dims = ::FFI::MemoryPointer.new(:int, num_values)
+              num_dims.write_array_of_int(attr_value.map(&:size))
+
+              FFI.TFE_OpSetAttrShapeList(op, attr_name, dims, num_dims, num_values, status)
+            when :type
+              values = ::FFI::MemoryPointer.new(:int, num_values)
+              types =
+                attr_value.map do |v|
+                  if v.is_a?(Symbol)
+                    FFI::DataType[v]
+                  else
+                    v
+                  end
+                end
+              values.write_array_of_int(types)
+              FFI.TFE_OpSetAttrTypeList(op, attr_name, values, num_values)
+            else
+              raise "Unknown list type: #{FFI::AttrType[type]}"
+            end
           else
-            raise "Unknown type: #{FFI::AttrType[type]}"
+            case FFI::AttrType[type]
+            when :string
+              FFI.TFE_OpSetAttrString(op, attr_name, attr_value, attr_value.bytesize)
+            when :int
+              FFI.TFE_OpSetAttrInt(op, attr_name, attr_value)
+            when :float
+              FFI.TFE_OpSetAttrFloat(op, attr_name, attr_value)
+            when :bool
+              FFI.TFE_OpSetAttrBool(op, attr_name, attr_value ? 1 : 0)
+            when :type
+              attr_value = FFI::DataType[attr_value] if attr_value.is_a?(Symbol)
+              FFI.TFE_OpSetAttrType(op, attr_name, attr_value)
+            when :shape
+              ptr = ::FFI::MemoryPointer.new(:int64, attr_value.size)
+              ptr.write_array_of_int64(attr_value)
+              FFI.TFE_OpSetAttrShape(op, attr_name, ptr, attr_value.size, status)
+              check_status status
+            # when :tensor
+            # when :placeholder
+            # when :func
+            else
+              raise "Unknown type: #{FFI::AttrType[type]}"
+            end
           end
         end
 
-        inputs.each do |input|
-          input = TensorFlow.convert_to_tensor(input) unless input.respond_to?(:to_ptr)
-          FFI.TFE_OpAddInput(op, input, status)
+        inputs.each_with_index do |input, i|
+          # TODO handle this better
+          if op_name == "TensorSliceDataset" && i == 0
+            input_ptr = ::FFI::MemoryPointer.new(:pointer, input.size)
+            input_ptr.write_array_of_pointer(input)
+            FFI.TFE_OpAddInputList(op, input_ptr, input.size, status)
+          else
+            raise "Missing argument" if input.nil?
+
+            input = TensorFlow.convert_to_tensor(input) unless input.respond_to?(:to_ptr)
+            FFI.TFE_OpAddInput(op, input, status)
+          end
           check_status status
         end
 
-        retvals = ::FFI::MemoryPointer.new(:pointer)
+        # TODO decide how many retvals to allocate
+        retvals = ::FFI::MemoryPointer.new(:pointer, 2)
         num_retvals = ::FFI::MemoryPointer.new(:int)
         num_retvals.write_int(retvals.size)
         FFI.TFE_Execute(op, retvals, num_retvals, status)
         check_status status
 
-        if num_retvals.read_int > 0
-          handle = retvals.read_pointer
-          type = FFI.TFE_TensorHandleDataType(handle)
+        n = num_retvals.read_int
+        if n > 0
+          retvals =
+            retvals.read_array_of_pointer(n).map do |handle|
+              Tensor.new(pointer: handle)
+            end
 
-          case FFI::DataType[type]
-          when :resource
-            handle
-          else
-            Tensor.new(pointer: handle)
-          end
+          # TODO handle case where n = 1 and still want an array for retvals
+          n == 1 ? retvals.first : retvals
         end
       ensure
         FFI.TF_DeleteStatus(status) if status
@@ -75,9 +144,18 @@ module TensorFlow
       end
 
       def infer_type(value)
-        if value.all? { |v| v.is_a?(String) }
+        if value.is_a?(Numo::NArray)
+          type = NUMO_TYPE_MAP.find { |k, v| value.is_a?(v) }
+          if type
+            type.first
+          else
+            raise Error, "Unable to infer data type"
+          end
+        elsif value.empty?
+          raise Error, "Unable to infer data type"
+        elsif value.all? { |v| v.is_a?(String) }
           :string
-        elsif value.all? { |v| v == true || v == false }
+        elsif value.all? { |v| v.is_a?(TrueClass) || v.is_a?(FalseClass) }
           :bool
         elsif value.all? { |v| v.is_a?(Integer) }
           if value.all? { |v| v >= -2147483648 && v <= 2147483647 }
@@ -94,47 +172,14 @@ module TensorFlow
         end
       end
 
-      def load_dataset(path, url)
-        # TODO handle this better
-        raise "No HOME" unless ENV["HOME"]
-        datasets_dir = "#{ENV["HOME"]}/.keras/datasets"
-        FileUtils.mkdir_p(datasets_dir)
-
-        path = "#{datasets_dir}/#{path}"
-        Utils.download_file(url, path) unless File.exist?(path)
-        Npy.load_npz(path)
-      end
-
-      def download_file(url, dest)
-        uri = URI(url)
-
-        temp_dir ||= File.dirname(Tempfile.new("tensorflow"))
-        temp_path = "#{temp_dir}/#{Time.now.to_f}" # TODO better name
-
-        # Net::HTTP automatically adds Accept-Encoding for compression
-        # of response bodies and automatically decompresses gzip
-        # and deflateresponses unless a Range header was sent.
-        # https://ruby-doc.org/stdlib-2.6.4/libdoc/net/http/rdoc/Net/HTTP.html
-        Net::HTTP.start(uri.host, uri.port, use_ssl: true) do |http|
-          request = Net::HTTP::Get.new(uri)
-
-          print("Downloading dataset")
-          i = 0
-          File.open(temp_path, "wb") do |f|
-            http.request(request) do |response|
-              response.read_body do |chunk|
-                f.write(chunk)
-
-                # print progress
-                putc "." if i % 50 == 0
-                i += 1
-              end
-            end
-            puts # newline
+      def to_tensor_array(values)
+        values.map do |v|
+          if v.is_a?(Tensor)
+            v
+          else
+            TensorFlow.convert_to_tensor(v)
           end
         end
-
-        FileUtils.mv(temp_path, dest)
       end
     end
   end