tensorflow 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 509b05a4a14e0bbb314c682f4172959b264e6709f92afc0fa0a749afccfe233b
-  data.tar.gz: daa060a1cb5c2602a652d87b75986a78a0d440c631b5fd7363e822363339f0ca
+  metadata.gz: eace7d05756f006a0710499d67eeb878234a5c67fafb877aa5e154f7f58a0b0b
+  data.tar.gz: 87f168e411872d952562550b6dfd73ed902c989b5d54df241198732f7a518ae5
 SHA512:
-  metadata.gz: '0755950b3b9865fe17a60efa970c23afcbbba33250b78e78bed7ddb907d90ecdfe64ffdd3fc48533ab6a30fbee6a64dbe6fd4d30e8554ca5c3f9535ea46e1daa'
-  data.tar.gz: 7307f84476cf789486a4f21328056a5abc5485a9dadeff99216dfa12f7a68c2b24f410fdb7caaca1c90a43d3497e03bb9459dbb876543819dce5e210afd02e3a
+  metadata.gz: cc67f8f76b77adc109f85c86aba641e71e6a52f60f7de9a0c983af9bd8a1ada5dc989ebfc8ae40b75da2957d24c07716385d6d949d1df7dc5fd88d4e7bb1172c
+  data.tar.gz: 545a0fb338134f3cd1088d6467cbaa3ebba2465a845bdb91ada369cbc594f5e63108d20a6d6d5e4884885a42b915483fce8414e0ae1ec27b0833b9861a176fe1

data/CHANGELOG.md

@@ -1,6 +1,13 @@
+## 0.1.2
+
+- Added `Audio`, `Bitwise`, `Image`, `IO`, `Linalg`, `NN`, `Strings`, and `Data::Dataset` modules
+- Added `numo` method to tensors
+- Added Boston housing dataset
+- Fixed string encoding
+
 ## 0.1.1
 
-- Added more operations
+- Added `Math` and `RawOps` modules
 - Added support for more data types
 - Added MNIST dataset
 

data/README.md

@@ -39,6 +39,13 @@ v = Tf::Variable.new(0.0)
 w = v + 1
 ```
 
+## Math
+
+```ruby
+Tf::Math.abs(-2)
+Tf::Math.sqrt(4)
+```
+
 ## FizzBuzz
 
 ```ruby
@@ -60,9 +67,24 @@ end
 fizzbuzz(15)
 ```
 
-## Keras
+## Data::Dataset
+
+```ruby
+# load
+train_dataset = Tf::Data::Dataset.from_tensor_slices([train_examples, train_labels])
+test_dataset = Tf::Data::Dataset.from_tensor_slices([test_examples, test_labels])
+
+# shuffle and batch
+train_dataset = train_dataset.shuffle(100).batch(32)
+test_dataset = test_dataset.batch(32)
 
-Coming soon
+# iterate
+train_dataset.each do |examples, labels|
+  # ...
+end
+```
+
+## Keras [coming soon]
 
 ```ruby
 mnist = Tf::Keras::Datasets::MNIST
@@ -114,3 +136,12 @@ Everyone is encouraged to help improve this project. Here are a few ways you can
 - Fix bugs and [submit pull requests](https://github.com/ankane/tensorflow/pulls)
 - Write, clarify, or fix documentation
 - Suggest or add new features
+
+To get started with development and testing:
+
+```sh
+git clone https://github.com/ankane/tensorflow.git
+cd tensorflow
+bundle install
+rake test
+```

data/lib/tensorflow.rb

@@ -1,29 +1,54 @@
 # dependencies
 require "ffi"
 require "npy"
+require "numo/narray"
 
 # stdlib
+require "digest"
 require "fileutils"
 require "forwardable"
+require "json"
 require "net/http"
 require "tempfile"
+require "zlib"
 
 # modules
-require "tensorflow/utils"
+require "tensorflow/audio"
+require "tensorflow/bitwise"
 require "tensorflow/context"
+require "tensorflow/image"
+require "tensorflow/io"
+require "tensorflow/linalg"
 require "tensorflow/math"
+require "tensorflow/nn"
 require "tensorflow/ops"
 require "tensorflow/raw_ops"
+require "tensorflow/strings"
 require "tensorflow/tensor"
+require "tensorflow/utils"
 require "tensorflow/variable"
 require "tensorflow/version"
 
+# data
+require "tensorflow/data/dataset"
+require "tensorflow/data/batch_dataset"
+require "tensorflow/data/shuffle_dataset"
+require "tensorflow/data/tensor_slice_dataset"
+
 # keras
+require "tensorflow/keras/datasets/boston_housing"
+require "tensorflow/keras/datasets/cifar10"
+require "tensorflow/keras/datasets/cifar100"
+require "tensorflow/keras/datasets/fashion_mnist"
+require "tensorflow/keras/datasets/imdb"
 require "tensorflow/keras/datasets/mnist"
+require "tensorflow/keras/datasets/reuters"
 require "tensorflow/keras/layers/dense"
 require "tensorflow/keras/layers/dropout"
 require "tensorflow/keras/layers/flatten"
+require "tensorflow/keras/metrics/mean"
 require "tensorflow/keras/models/sequential"
+require "tensorflow/keras/utils"
 
 module TensorFlow
   class Error < StandardError; end
@@ -41,7 +66,9 @@ module TensorFlow
     include Utils
 
     extend Forwardable
+    def_delegators Linalg, :eye, :matmul
     def_delegators Math, :abs, :acos, :acosh, :add, :add_n, :argmax, :argmin, :asin, :asinh, :atan, :atan2, :atanh, :cos, :cosh, :cumsum, :divide, :equal, :exp, :floor, :greater, :greater_equal, :less, :less_equal, :logical_and, :logical_not, :logical_or, :maximum, :minimum, :multiply, :negative, :not_equal, :pow, :reduce_all, :reduce_any, :reduce_logsumexp, :reduce_max, :reduce_mean, :reduce_min, :reduce_prod, :reduce_sum, :round, :scalar_mul, :sigmoid, :sign, :sin, :sinh, :sqrt, :square, :subtract, :tan, :tanh, :truediv
+    def_delegators NN, :space_to_batch
 
     def library_version
       FFI.TF_Version

data/lib/tensorflow/audio.rb

@@ -0,0 +1,13 @@
+module TensorFlow
+  module Audio
+    class << self
+      def decode_wav(contents, desired_channels: -1, desired_samples: -1)
+        RawOps.decode_wav(contents: contents, desired_channels: desired_channels, desired_samples: desired_samples)
+      end
+
+      def encode_wav(audio, sample_rate)
+        RawOps.encode_wav(audio: audio, sample_rate: sample_rate)
+      end
+    end
+  end
+end

data/lib/tensorflow/bitwise.rb

@@ -0,0 +1,29 @@
+module TensorFlow
+  module Bitwise
+    class << self
+      def bitwise_and(x, y)
+        RawOps.bitwise_and(x: x, y: y)
+      end
+
+      def bitwise_or(x, y)
+        RawOps.bitwise_or(x: x, y: y)
+      end
+
+      def bitwise_xor(x, y)
+        RawOps.bitwise_xor(x: x, y: y)
+      end
+
+      def invert(x)
+        RawOps.invert(x: x)
+      end
+
+      def left_shift(x, y)
+        RawOps.left_shift(x: x, y: y)
+      end
+
+      def right_shift(x, y)
+        RawOps.right_shift(x: x, y: y)
+      end
+    end
+  end
+end

data/lib/tensorflow/data/batch_dataset.rb

@@ -0,0 +1,20 @@
+module TensorFlow
+  module Data
+    class BatchDataset < Dataset
+      def initialize(input_dataset, batch_size, drop_remainder)
+        @input_dataset = input_dataset # keep reference for memory
+        @output_types = input_dataset.output_types
+        @output_shapes = input_dataset.output_shapes.map { |s| [batch_size] + s }
+
+        variant_tensor = RawOps.batch_dataset_v2(
+          input_dataset: input_dataset,
+          batch_size: TensorFlow.convert_to_tensor(batch_size, dtype: :int64),
+          drop_remainder: drop_remainder,
+          output_types: @output_types,
+          output_shapes: @output_shapes
+        )
+        super(variant_tensor)
+      end
+    end
+  end
+end

data/lib/tensorflow/data/dataset.rb

@@ -0,0 +1,46 @@
+module TensorFlow
+  module Data
+    class Dataset
+      include Enumerable
+
+      # TODO remove
+      attr_reader :output_types, :output_shapes
+
+      def initialize(variant_tensor)
+        @variant_tensor = variant_tensor
+      end
+
+      def batch(batch_size, drop_remainder: false)
+        BatchDataset.new(self, batch_size, drop_remainder)
+      end
+
+      def shuffle(buffer_size)
+        ShuffleDataset.new(self, buffer_size)
+      end
+
+      def self.from_tensor_slices(tensors)
+        TensorSliceDataset.new(tensors)
+      end
+
+      def to_ptr
+        @variant_tensor.to_ptr
+      end
+
+      def each
+        iterator, deleter = RawOps.anonymous_iterator_v2(output_types: @output_types, output_shapes: @output_shapes)
+        RawOps.make_iterator(dataset: @variant_tensor, iterator: iterator)
+        begin
+          loop do
+            values = RawOps.iterator_get_next_sync(iterator: iterator, output_types: @output_types, output_shapes: @output_shapes)
+            yield values
+          end
+        rescue Error => e
+          # iterate until end of sequence error
+          raise e unless e.message == "End of sequence"
+        end
+      ensure
+        RawOps.delete_iterator(handle: iterator, deleter: deleter) if iterator
+      end
+    end
+  end
+end

data/lib/tensorflow/data/shuffle_dataset.rb

@@ -0,0 +1,21 @@
+module TensorFlow
+  module Data
+    class ShuffleDataset < Dataset
+      def initialize(input_dataset, buffer_size)
+        @input_dataset = input_dataset # keep reference for memory
+        @output_types = input_dataset.output_types
+        @output_shapes = input_dataset.output_shapes
+
+        variant_tensor = RawOps.shuffle_dataset(
+          input_dataset: input_dataset,
+          buffer_size: TensorFlow.convert_to_tensor(buffer_size, dtype: :int64),
+          seed: TensorFlow.convert_to_tensor(0, dtype: :int64),
+          seed2: TensorFlow.convert_to_tensor(0, dtype: :int64),
+          output_types: @output_types,
+          output_shapes: @output_shapes
+        )
+        super(variant_tensor)
+      end
+    end
+  end
+end

data/lib/tensorflow/data/tensor_slice_dataset.rb

@@ -0,0 +1,15 @@
+module TensorFlow
+  module Data
+    class TensorSliceDataset < Dataset
+      def initialize(element)
+        tensors = Utils.to_tensor_array(element)
+        @tensors = tensors # keep reference for memory
+        @output_types = tensors.map(&:dtype)
+        @output_shapes = tensors.map { |t| t.shape[1..-1] }
+
+        variant_tensor = RawOps.tensor_slice_dataset(components: tensors, output_shapes: @output_shapes)
+        super(variant_tensor)
+      end
+    end
+  end
+end

data/lib/tensorflow/ffi.rb

@@ -35,6 +35,10 @@ module TensorFlow
     attach_function :TF_NewTensor, %i[int pointer int pointer size_t pointer pointer], :pointer
     attach_function :TF_DeleteTensor, %i[pointer], :void
     attach_function :TF_TensorData, %i[pointer], :pointer
+    attach_function :TF_TensorByteSize, %i[pointer], :size_t
+    attach_function :TF_StringEncode, %i[pointer size_t pointer size_t pointer], :size_t
+    attach_function :TF_StringDecode, %i[pointer size_t pointer pointer pointer], :size_t
+    attach_function :TF_StringEncodedSize, %i[size_t], :size_t
 
     # https://github.com/tensorflow/tensorflow/blob/master/tensorflow/c/eager/c_api.h
     attach_function :TFE_NewContextOptions, %i[], :pointer
@@ -58,10 +62,18 @@ module TensorFlow
     attach_function :TFE_OpSetDevice, %i[pointer string pointer], :pointer
     attach_function :TFE_OpGetDevice, %i[pointer pointer], :string
     attach_function :TFE_OpAddInput, %i[pointer pointer pointer], :void
+    attach_function :TFE_OpAddInputList, %i[pointer pointer int pointer], :void
     attach_function :TFE_OpGetAttrType, %i[pointer string pointer pointer], :int
     attach_function :TFE_OpSetAttrString, %i[pointer string pointer size_t], :void
+    attach_function :TFE_OpSetAttrInt, %i[pointer string int64_t], :void
+    attach_function :TFE_OpSetAttrFloat, %i[pointer string float], :void
+    attach_function :TFE_OpSetAttrBool, %i[pointer string uint8], :void
     attach_function :TFE_OpSetAttrType, %i[pointer string int], :void
     attach_function :TFE_OpSetAttrShape, %i[pointer string pointer int pointer], :void
+    attach_function :TFE_OpSetAttrIntList, %i[pointer string pointer int], :void
+    attach_function :TFE_OpSetAttrFloatList, %i[pointer string pointer int], :void
+    attach_function :TFE_OpSetAttrTypeList, %i[pointer string pointer int], :void
+    attach_function :TFE_OpSetAttrShapeList, %i[pointer string pointer pointer int pointer], :void
     attach_function :TFE_Execute, %i[pointer pointer pointer pointer], :pointer
   end
 end

data/lib/tensorflow/image.rb

@@ -0,0 +1,218 @@
+module TensorFlow
+  module Image
+    class << self
+      # def adjust_brightness
+      # end
+
+      def adjust_contrast(images, contrast_factor, min_value, max_value)
+        RawOps.adjust_contrast(images: images, contrast_factor: contrast_factor, min_value: min_value, max_value: max_value)
+      end
+
+      # def adjust_gamma
+      # end
+
+      def adjust_hue(images, delta)
+        RawOps.adjust_hue(images: images, delta: delta)
+      end
+
+      # def adjust_jpeg_quality
+      # end
+
+      def adjust_saturation(images, scale)
+        RawOps.adjust_saturation(images: images, scale: scale)
+      end
+
+      # def central_crop
+      # end
+
+      def combined_non_max_suppression(boxes, scores, max_output_size_per_class, max_total_size, iou_threshold, score_threshold, pad_per_class: nil, clip_boxes: nil)
+        RawOps.combined_non_max_suppression(boxes: boxes, scores: scores, max_output_size_per_class: max_output_size_per_class, max_total_size: max_total_size, iou_threshold: iou_threshold, score_threshold: score_threshold, pad_per_class: pad_per_class, clip_boxes: clip_boxes)
+      end
+
+      # def convert_image_dtype
+      # end
+
+      def crop_and_resize(image, boxes, box_ind, crop_size, method: nil, extrapolation_value: nil)
+        RawOps.crop_and_resize(image: image, boxes: boxes, box_ind: box_ind, crop_size: crop_size, method: method, extrapolation_value: extrapolation_value)
+      end
+
+      # def crop_to_bounding_box
+      # end
+
+      def decode_and_crop_jpeg(contents, crop_window, channels: nil, ratio: nil, fancy_upscaling: nil, try_recover_truncated: nil, acceptable_fraction: nil, dct_method: nil)
+        RawOps.decode_and_crop_jpeg(contents: contents, crop_window: crop_window, channels: channels, ratio: ratio, fancy_upscaling: fancy_upscaling, try_recover_truncated: try_recover_truncated, acceptable_fraction: acceptable_fraction, dct_method: dct_method)
+      end
+
+      def decode_bmp(contents, channels: nil)
+        RawOps.decode_bmp(contents: contents, channels: channels)
+      end
+
+      def decode_gif(contents)
+        RawOps.decode_gif(contents: contents)
+      end
+
+      # def decode_image
+      # end
+
+      def decode_jpeg(contents, channels: nil, ratio: nil, fancy_upscaling: nil, try_recover_truncated: nil, acceptable_fraction: nil, dct_method: nil)
+        RawOps.decode_jpeg(contents: contents, channels: channels, ratio: ratio, fancy_upscaling: fancy_upscaling, try_recover_truncated: try_recover_truncated, acceptable_fraction: acceptable_fraction, dct_method: dct_method)
+      end
+
+      def decode_png(contents, channels: nil, dtype: nil)
+        RawOps.decode_png(contents: contents, channels: channels, dtype: dtype)
+      end
+
+      def draw_bounding_boxes(images, boxes)
+        RawOps.draw_bounding_boxes(images: images, boxes: boxes)
+      end
+
+      def encode_jpeg(image, format: nil, quality: nil, progressive: nil, optimize_size: nil, chroma_downsampling: nil, density_unit: nil, x_density: nil, y_density: nil, xmp_metadata: nil)
+        RawOps.encode_jpeg(image: image, format: format, quality: quality, progressive: progressive, optimize_size: optimize_size, chroma_downsampling: chroma_downsampling, density_unit: density_unit, x_density: x_density, y_density: y_density, xmp_metadata: xmp_metadata)
+      end
+
+      def encode_png(image, compression: nil)
+        RawOps.encode_png(image: image, compression: compression)
+      end
+
+      def extract_glimpse(input, size, offsets, centered: nil, normalized: nil, uniform_noise: nil, noise: nil)
+        RawOps.extract_glimpse(input: input, size: size, offsets: offsets, centered: centered, normalized: normalized, uniform_noise: uniform_noise, noise: noise)
+      end
+
+      def extract_jpeg_shape(contents, output_type: nil)
+        RawOps.extract_jpeg_shape(contents: contents, output_type: output_type)
+      end
+
+      # def extract_patches
+      # end
+
+      # def flip_left_right
+      # end
+
+      # def flip_up_down
+      # end
+
+      # def grayscale_to_rgb
+      # end
+
+      def hsv_to_rgb(images)
+        RawOps.hsv_to_rgb(images: images)
+      end
+
+      # def image_gradients
+      # end
+
+      def is_jpeg(contents)
+        substr = Strings.substr(contents, 0, 3)
+        Math.equal(substr, "\xff\xd8\xff")
+      end
+
+      def non_max_suppression(boxes, scores, max_output_size, iou_threshold: nil)
+        RawOps.non_max_suppression(boxes: boxes, scores: scores, max_output_size: max_output_size, iou_threshold: iou_threshold)
+      end
+
+      # def non_max_suppression_overlaps
+      # end
+
+      # def non_max_suppression_padded
+      # end
+
+      # def non_max_suppression_with_scores
+      # end
+
+      # def pad_to_bounding_box
+      # end
+
+      # def per_image_standardization
+      # end
+
+      # def psnr
+      # end
+
+      # def random_brightness
+      # end
+
+      # def random_contrast
+      # end
+
+      def random_crop(image, size, seed: nil, seed2: nil)
+        RawOps.random_crop(image: image, size: size, seed: seed, seed2: seed2)
+      end
+
+      # def random_flip_left_right
+      # end
+
+      # def random_flip_up_down
+      # end
+
+      # def random_hue
+      # end
+
+      # def random_jpeg_quality
+      # end
+
+      # def random_saturation
+      # end
+
+      def resize(images, size)
+        images = TensorFlow.convert_to_tensor(images)
+
+        batch = images.shape.size != 3
+        images = TensorFlow.expand_dims(images, 0) unless batch
+
+        # TODO support more methods
+        images = RawOps.resize_bilinear(images: images, size: size)
+
+        images = TensorFlow.squeeze(images, axis: [0]) unless batch
+
+        images
+      end
+
+      # def resize_with_crop_or_pad
+      # end
+
+      # def resize_with_pad
+      # end
+
+      # def rgb_to_grayscale
+      # end
+
+      def rgb_to_hsv(images)
+        RawOps.rgb_to_hsv(images: images)
+      end
+
+      # def rgb_to_yiq
+      # end
+
+      # def rgb_to_yuv
+      # end
+
+      # def rot90
+      # end
+
+      def sample_distorted_bounding_box(image_size, bounding_boxes, seed: nil, seed2: nil, min_object_covered: nil, aspect_ratio_range: nil, area_range: nil, max_attempts: nil, use_image_if_no_bounding_boxes: nil)
+        RawOps.sample_distorted_bounding_box(image_size: image_size, bounding_boxes: bounding_boxes, seed: seed, seed2: seed2, min_object_covered: min_object_covered, aspect_ratio_range: aspect_ratio_range, area_range: area_range, max_attempts: max_attempts, use_image_if_no_bounding_boxes: use_image_if_no_bounding_boxes)
+      end
+
+      # def sobel_edges
+      # end
+
+      # def ssim
+      # end
+
+      # def ssim_multiscale
+      # end
+
+      # def total_variation
+      # end
+
+      # def transpose
+      # end
+
+      # def yiq_to_rgb
+      # end
+
+      # def yuv_to_rgb
+      # end
+    end
+  end
+end