tensorflow 0.1.1 → 0.1.2

data/lib/tensorflow/io.rb

@@ -0,0 +1,125 @@
+module TensorFlow
+  module IO
+    class << self
+      def decode_and_crop_jpeg(contents, crop_window, channels: nil, ratio: nil, fancy_upscaling: nil, try_recover_truncated: nil, acceptable_fraction: nil, dct_method: nil)
+        RawOps.decode_and_crop_jpeg(contents: contents, crop_window: crop_window, channels: channels, ratio: ratio, fancy_upscaling: fancy_upscaling, try_recover_truncated: try_recover_truncated, acceptable_fraction: acceptable_fraction, dct_method: dct_method)
+      end
+
+      def decode_base64(input)
+        RawOps.decode_base64(input: input)
+      end
+
+      def decode_bmp(contents, channels: nil)
+        RawOps.decode_bmp(contents: contents, channels: channels)
+      end
+
+      def decode_compressed(bytes, compression_type: nil)
+        RawOps.decode_compressed(bytes: bytes, compression_type: compression_type)
+      end
+
+      def decode_csv(records, record_defaults, field_delim: nil, use_quote_delim: nil, na_value: nil, select_cols: nil)
+        RawOps.decode_csv(records: records, record_defaults: record_defaults, field_delim: field_delim, use_quote_delim: use_quote_delim, na_value: na_value, select_cols: select_cols)
+      end
+
+      def decode_gif(contents)
+        RawOps.decode_gif(contents: contents)
+      end
+
+      # def decode_image
+      # end
+
+      def decode_jpeg(contents, channels: nil, ratio: nil, fancy_upscaling: nil, try_recover_truncated: nil, acceptable_fraction: nil, dct_method: nil)
+        RawOps.decode_jpeg(contents: contents, channels: channels, ratio: ratio, fancy_upscaling: fancy_upscaling, try_recover_truncated: try_recover_truncated, acceptable_fraction: acceptable_fraction, dct_method: dct_method)
+      end
+
+      def decode_json_example(json_examples)
+        RawOps.decode_json_example(json_examples: json_examples)
+      end
+
+      def decode_png(contents, channels: nil, dtype: nil)
+        RawOps.decode_png(contents: contents, channels: channels, dtype: dtype)
+      end
+
+      # def decode_proto
+      # end
+
+      def decode_raw(bytes, out_type: nil, little_endian: nil)
+        RawOps.decode_raw(bytes: bytes, out_type: out_type, little_endian: little_endian)
+      end
+
+      def deserialize_many_sparse(serialized_sparse, dtype: nil)
+        RawOps.deserialize_many_sparse(serialized_sparse: serialized_sparse, dtype: dtype)
+      end
+
+      def encode_base64(input, pad: nil)
+        RawOps.encode_base64(input: input, pad: pad)
+      end
+
+      def encode_jpeg(image, format: nil, quality: nil, progressive: nil, optimize_size: nil, chroma_downsampling: nil, density_unit: nil, x_density: nil, y_density: nil, xmp_metadata: nil)
+        RawOps.encode_jpeg(image: image, format: format, quality: quality, progressive: progressive, optimize_size: optimize_size, chroma_downsampling: chroma_downsampling, density_unit: density_unit, x_density: x_density, y_density: y_density, xmp_metadata: xmp_metadata)
+      end
+
+      def encode_proto(sizes, values, field_names: nil, message_type: nil, descriptor_source: nil)
+        RawOps.encode_proto(sizes: sizes, values: values, field_names: field_names, message_type: message_type, descriptor_source: descriptor_source)
+      end
+
+      def extract_jpeg_shape(contents, output_type: nil)
+        RawOps.extract_jpeg_shape(contents: contents, output_type: output_type)
+      end
+
+      def is_jpeg(contents)
+        Image.is_jpeg(contents)
+      end
+
+      # def match_filenames_once
+      # end
+
+      def matching_files(pattern)
+        RawOps.matching_files(pattern: pattern)
+      end
+
+      def parse_example(serialized, names, sparse_keys, dense_keys, dense_defaults, sparse_types: nil, dense_shapes: nil)
+        RawOps.parse_example(serialized: serialized, names: names, sparse_keys: sparse_keys, dense_keys: dense_keys, dense_defaults: dense_defaults, sparse_types: sparse_types, dense_shapes: dense_shapes)
+      end
+
+      def parse_sequence_example(serialized, debug_name, context_dense_defaults, feature_list_dense_missing_assumed_empty: nil, context_sparse_keys: nil, context_dense_keys: nil, feature_list_sparse_keys: nil, feature_list_dense_keys: nil, context_sparse_types: nil, feature_list_dense_types: nil, context_dense_shapes: nil, feature_list_sparse_types: nil, feature_list_dense_shapes: nil)
+        RawOps.parse_sequence_example(serialized: serialized, debug_name: debug_name, context_dense_defaults: context_dense_defaults, feature_list_dense_missing_assumed_empty: feature_list_dense_missing_assumed_empty, context_sparse_keys: context_sparse_keys, context_dense_keys: context_dense_keys, feature_list_sparse_keys: feature_list_sparse_keys, feature_list_dense_keys: feature_list_dense_keys, context_sparse_types: context_sparse_types, feature_list_dense_types: feature_list_dense_types, context_dense_shapes: context_dense_shapes, feature_list_sparse_types: feature_list_sparse_types, feature_list_dense_shapes: feature_list_dense_shapes)
+      end
+
+      def parse_single_example(serialized, dense_defaults, num_sparse: nil, sparse_keys: nil, dense_keys: nil, sparse_types: nil, dense_shapes: nil)
+        RawOps.parse_single_example(serialized: serialized, dense_defaults: dense_defaults, num_sparse: num_sparse, sparse_keys: sparse_keys, dense_keys: dense_keys, sparse_types: sparse_types, dense_shapes: dense_shapes)
+      end
+
+      def parse_single_sequence_example(serialized, feature_list_dense_missing_assumed_empty, context_sparse_keys, context_dense_keys, feature_list_sparse_keys, feature_list_dense_keys, context_dense_defaults, debug_name, context_sparse_types: nil, feature_list_dense_types: nil, context_dense_shapes: nil, feature_list_sparse_types: nil, feature_list_dense_shapes: nil)
+        RawOps.parse_single_sequence_example(serialized: serialized, feature_list_dense_missing_assumed_empty: feature_list_dense_missing_assumed_empty, context_sparse_keys: context_sparse_keys, context_dense_keys: context_dense_keys, feature_list_sparse_keys: feature_list_sparse_keys, feature_list_dense_keys: feature_list_dense_keys, context_dense_defaults: context_dense_defaults, debug_name: debug_name, context_sparse_types: context_sparse_types, feature_list_dense_types: feature_list_dense_types, context_dense_shapes: context_dense_shapes, feature_list_sparse_types: feature_list_sparse_types, feature_list_dense_shapes: feature_list_dense_shapes)
+      end
+
+      def parse_tensor(serialized, out_type: nil)
+        RawOps.parse_tensor(serialized: serialized, out_type: out_type)
+      end
+
+      def read_file(filename)
+        RawOps.read_file(filename: filename)
+      end
+
+      def serialize_many_sparse(sparse_indices, sparse_values, sparse_shape, out_type: nil)
+        RawOps.serialize_many_sparse(sparse_indices: sparse_indices, sparse_values: sparse_values, sparse_shape: sparse_shape, out_type: out_type)
+      end
+
+      def serialize_sparse(sparse_indices, sparse_values, sparse_shape, out_type: nil)
+        RawOps.serialize_sparse(sparse_indices: sparse_indices, sparse_values: sparse_values, sparse_shape: sparse_shape, out_type: out_type)
+      end
+
+      def serialize_tensor(tensor)
+        RawOps.serialize_tensor(tensor: tensor)
+      end
+
+      def write_file(filename, contents)
+        RawOps.write_file(filename: filename, contents: contents)
+      end
+
+      # def write_graph
+      # end
+    end
+  end
+end

data/lib/tensorflow/keras/datasets/boston_housing.rb

@@ -0,0 +1,32 @@
+module TensorFlow
+  module Keras
+    module Datasets
+      module BostonHousing
+        def self.load_data(path: "boston_housing.npz", test_split: 0.2, seed: 113)
+          file = Utils.get_file(
+            path,
+            "https://storage.googleapis.com/tensorflow/tf-keras-datasets/boston_housing.npz",
+            file_hash: "f553886a1f8d56431e820c5b82552d9d95cfcb96d1e678153f8839538947dff5"
+          )
+
+          data = Npy.load_npz(file)
+
+          x = data["x"]
+          y = data["y"]
+
+          len = x.shape[0]
+          indices = (0...len).to_a.shuffle(random: Random.new(seed))
+          x = x[indices, true]
+          y = y[indices]
+
+          x_train = x[0...(len * (1 - test_split)), true]
+          y_train = y[0...(len * (1 - test_split))]
+          x_test = x[(len * (1 - test_split))..-1, true]
+          y_test = y[(len * (1 - test_split))..-1]
+
+          [[x_train, y_train], [x_test, y_test]]
+        end
+      end
+    end
+  end
+end

data/lib/tensorflow/keras/datasets/cifar10.rb

@@ -0,0 +1,11 @@
+module TensorFlow
+  module Keras
+    module Datasets
+      module CIFAR10
+        # unfortunately, npy can't read pickle and numo can't store objects
+        # def self.load_data
+        # end
+      end
+    end
+  end
+end

data/lib/tensorflow/keras/datasets/cifar100.rb

@@ -0,0 +1,11 @@
+module TensorFlow
+  module Keras
+    module Datasets
+      module CIFAR100
+        # unfortunately, npy can't read pickle and numo can't store objects
+        # def self.load_data
+        # end
+      end
+    end
+  end
+end

data/lib/tensorflow/keras/datasets/fashion_mnist.rb

@@ -0,0 +1,44 @@
+module TensorFlow
+  module Keras
+    module Datasets
+      module FashionMNIST
+        def self.load_data
+          base_url = "https://storage.googleapis.com/tensorflow/tf-keras-datasets"
+          files = [
+            "train-labels-idx1-ubyte.gz", "train-images-idx3-ubyte.gz",
+            "t10k-labels-idx1-ubyte.gz", "t10k-images-idx3-ubyte.gz"
+          ]
+
+          paths = []
+          files.each do |file|
+            paths << Utils.get_file(file, "#{base_url}/#{file}", cache_subdir: "datasets/fashion-mnist")
+          end
+
+          x_train, y_train, x_test, y_test = nil
+
+          Zlib::GzipReader.open(paths[0]) do |gz|
+            gz.read(8) # move to offset
+            y_train = Numo::UInt8.from_string(gz.read)
+          end
+
+          Zlib::GzipReader.open(paths[1]) do |gz|
+            gz.read(16) # move to offset
+            x_train = Numo::UInt8.from_string(gz.read, [y_train.shape[0], 28, 28])
+          end
+
+          Zlib::GzipReader.open(paths[2]) do |gz|
+            gz.read(8) # move to offset
+            y_test = Numo::UInt8.from_string(gz.read)
+          end
+
+          Zlib::GzipReader.open(paths[3]) do |gz|
+            gz.read(16) # move to offset
+            x_test = Numo::UInt8.from_string(gz.read, [y_test.shape[0], 28, 28])
+          end
+
+          [[x_train, y_train], [x_test, y_test]]
+        end
+      end
+    end
+  end
+end

data/lib/tensorflow/keras/datasets/imdb.rb

@@ -0,0 +1,30 @@
+module TensorFlow
+  module Keras
+    module Datasets
+      module IMDB
+        # unfortunately, npy can't read pickle and numo can't store objects
+        # def self.load_data(path: "imdb.npz", seed: 113)
+        #   data = Utils.load_dataset(
+        #     path,
+        #     "https://storage.googleapis.com/tensorflow/tf-keras-datasets/imdb.npz",
+        #     "69664113be75683a8fe16e3ed0ab59fda8886cb3cd7ada244f7d9544e4676b9f"
+        #   )
+
+        #   x_train = data["x_train"]
+        #   labels_train = data["y_train"]
+        #   x_test = data["x_test"]
+        #   labels_test = data["y_test"]
+        # end
+
+        def self.get_word_index(path: "imdb_word_index.json")
+          file = Utils.get_file(
+            path,
+            "https://storage.googleapis.com/tensorflow/tf-keras-datasets/imdb_word_index.json",
+            file_hash: "bfafd718b763782e994055a2d397834f"
+          )
+          JSON.parse(File.read(file))
+        end
+      end
+    end
+  end
+end

data/lib/tensorflow/keras/datasets/mnist.rb

@@ -2,13 +2,14 @@ module TensorFlow
   module Keras
     module Datasets
       module MNIST
-        def self.load_data
-          # TODO verify hash
-          data = Utils.load_dataset(
-            "mnist.npz",
-            "https://storage.googleapis.com/tensorflow/tf-keras-datasets/mnist.npz"
+        def self.load_data(path: "mnist.npz")
+          file = Utils.get_file(
+            path,
+            "https://storage.googleapis.com/tensorflow/tf-keras-datasets/mnist.npz",
+            file_hash: "731c5ac602752760c8e48fbffcf8c3b850d9dc2a2aedcf2cc48468fc17b673d1"
           )
 
+          data = Npy.load_npz(file)
           [[data["x_train"], data["y_train"]], [data["x_test"], data["y_test"]]]
         end
       end

data/lib/tensorflow/keras/datasets/reuters.rb

@@ -0,0 +1,28 @@
+module TensorFlow
+  module Keras
+    module Datasets
+      module Reuters
+        # unfortunately, npy can't read pickle and numo can't store objects
+        # def self.load_data(path: "reuters.npz", test_split: 0.2, seed: 113)
+        #   data = Utils.load_dataset(
+        #     path,
+        #     "https://storage.googleapis.com/tensorflow/tf-keras-datasets/reuters.npz",
+        #     "d6586e694ee56d7a4e65172e12b3e987c03096cb01eab99753921ef915959916"
+        #   )
+
+        #   xs = data["x"]
+        #   labels = data["y"]
+        # end
+
+        def self.get_word_index(path: "reuters_word_index.json")
+          file = Utils.get_file(
+            path,
+            "https://storage.googleapis.com/tensorflow/tf-keras-datasets/reuters_word_index.json",
+            file_hash: "4d44cc38712099c9e383dc6e5f11a921"
+          )
+          JSON.parse(File.read(file))
+        end
+      end
+    end
+  end
+end

data/lib/tensorflow/keras/metrics/mean.rb

@@ -0,0 +1,17 @@
+module TensorFlow
+  module Keras
+    module Metrics
+      class Mean
+        def update_state(values)
+          input = TensorFlow.convert_to_tensor(values, dtype: :float)
+          @total = Math.reduce_sum(input)
+          @count = RawOps.size(input: input)
+        end
+
+        def result
+          RawOps.div_no_nan(x: @total, y: TensorFlow.cast(@count, :float))
+        end
+      end
+    end
+  end
+end

data/lib/tensorflow/keras/utils.rb

@@ -0,0 +1,56 @@
+module TensorFlow
+  module Keras
+    module Utils
+      class << self
+        def get_file(fname, origin, file_hash: nil, cache_subdir: "datasets")
+          # destination
+          # TODO handle this better
+          raise "No HOME" unless ENV["HOME"]
+          dest = "#{ENV["HOME"]}/.keras/#{cache_subdir}/#{fname}"
+          FileUtils.mkdir_p(File.dirname(dest))
+
+          return dest if File.exist?(dest)
+
+          temp_dir ||= File.dirname(Tempfile.new("tensorflow"))
+          temp_path = "#{temp_dir}/#{Time.now.to_f}" # TODO better name
+
+          digest = file_hash&.size == 32 ? Digest::MD5.new : Digest::SHA2.new
+
+          uri = URI(origin)
+
+          # Net::HTTP automatically adds Accept-Encoding for compression
+          # of response bodies and automatically decompresses gzip
+          # and deflateresponses unless a Range header was sent.
+          # https://ruby-doc.org/stdlib-2.6.4/libdoc/net/http/rdoc/Net/HTTP.html
+          Net::HTTP.start(uri.host, uri.port, use_ssl: true) do |http|
+            request = Net::HTTP::Get.new(uri)
+
+            puts "Downloading data from #{origin}"
+            i = 0
+            File.open(temp_path, "wb") do |f|
+              http.request(request) do |response|
+                response.read_body do |chunk|
+                  f.write(chunk)
+                  digest.update(chunk)
+
+                  # print progress
+                  putc "." if i % 50 == 0
+                  i += 1
+                end
+              end
+              puts # newline
+            end
+          end
+
+          if file_hash && digest.hexdigest != file_hash
+            raise Error, "Bad hash"
+          end
+
+          FileUtils.mv(temp_path, dest)
+
+          dest
+        end
+      end
+    end
+  end
+end

data/lib/tensorflow/linalg.rb

@@ -0,0 +1,133 @@
+module TensorFlow
+  module Linalg
+    class << self
+      # def adjoint
+      # end
+
+      # def band_part
+      # end
+
+      def cholesky(input)
+        RawOps.cholesky(input: input)
+      end
+
+      # def cholesky_solve
+      # end
+
+      def cross(a, b)
+        RawOps.cross(a: a, b: b)
+      end
+
+      # def det
+      # end
+
+      def diag(diagonal)
+        RawOps.diag(diagonal: diagonal)
+      end
+
+      def diag_part(input)
+        RawOps.diag_part(input: input)
+      end
+
+      # def eigh
+      # end
+
+      # def eigvalsh
+      # end
+
+      # def einsum
+      # end
+
+      # def expm
+      # end
+
+      def eye(num_rows, num_columns: nil)
+        num_columns ||= num_rows
+        zeros = TensorFlow.zeros([num_rows, num_columns])
+        ones = TensorFlow.ones([num_rows])
+        RawOps.matrix_set_diag(input: zeros, diagonal: ones)
+      end
+
+      # def global_norm
+      # end
+
+      def inv(x)
+        RawOps.inv(x: x)
+      end
+
+      # def l2_normalize
+      # end
+
+      # def logdet
+      # end
+
+      # def logm
+      # end
+
+      # def lstsq
+      # end
+
+      def lu(input, output_idx_type: nil)
+        RawOps.lu(input: input, output_idx_type: output_idx_type)
+      end
+
+      def matmul(a, b, transpose_a: false, transpose_b: false)
+        RawOps.mat_mul(a: a, b: b, transpose_a: transpose_a, transpose_b: transpose_b)
+      end
+
+      # def matrix_transpose
+      # end
+
+      # def matvec
+      # end
+
+      # def norm
+      # end
+
+      # def normalize
+      # end
+
+      def qr(input, full_matrices: nil)
+        RawOps.qr(input: input, full_matrices: full_matrices)
+      end
+
+      # def set_diag
+      # end
+
+      # def slogdet
+      # end
+
+      # def solve
+      # end
+
+      # def sqrtm
+      # end
+
+      def svd(input, compute_uv: nil, full_matrices: nil)
+        RawOps.svd(input: input, compute_uv: compute_uv, full_matrices: full_matrices)
+      end
+
+      # def tensor_diag
+      # end
+
+      # def tensor_diag_part
+      # end
+
+      # def tensordot
+      # end
+
+      # def trace
+      # end
+
+      # def triangular_solve
+      # end
+
+      # def tridiagonal_matmul
+      # end
+
+      def tridiagonal_solve(diagonals, rhs, partial_pivoting: nil)
+        RawOps.tridiagonal_solve(diagonals: diagonals, rhs: rhs, partial_pivoting: partial_pivoting)
+      end
+    end
+  end
+end