treyja 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9ad4ca8e455a01ba5b5c44cf4314011f89e10f7ab48ee3c2ba88c08e7ab31105
-  data.tar.gz: c3bb03cf7e18d52ca9e6036903db316e9c7a672cafe346b42875d5f77deec31a
+  metadata.gz: 126534d5a966eec091dd2542c8e7fe0d70e5c483ca42324dba7872504976545b
+  data.tar.gz: 94e2b98f05650a6d0212ed795a43864797bf357ffaad6227537f6b3b44c768df
 SHA512:
-  metadata.gz: 7220d6f7efb6c80d3ab9bcd2778261b6095cf56dd72601fea2b5e9828456bd5ccf6c1fff843f0feb60e4701f639b459144b8523b6f3afaee0672afa344acab94
-  data.tar.gz: 172a497f871142f62aa488e08c5a3d210e0774dd1d5285d49ae8136f8b261235c0c4c058527f4f162b926e18daa1d3c1ba0d89ea4afc86b90eb4573921c8191c
+  metadata.gz: f8a4bfb3224a2e2ebf0b5dacb9263e75e281487411c6edde32f61a5464f4969773448dad0e309e27295e628082aac62833a6ed893317352949bd3c30ea71cfd2
+  data.tar.gz: 2cd1465e1e3cd403ba74c2b260c6d7cac9c20da4d072828f034975cd722fd30596e66ed8f3040e5216edc75d28c828ff1cb6f9cbeac019a48c58fb2f663fd41f

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    treyja (0.1.2)
+    treyja (0.1.3)
       google-protobuf (~> 3.6)
 
 GEM
@@ -9,7 +9,7 @@ GEM
   specs:
     coderay (1.1.2)
     diff-lcs (1.3)
-    google-protobuf (3.7.1)
+    google-protobuf (3.7.1-universal-darwin)
     method_source (0.9.2)
     pry (0.12.0)
       coderay (~> 1.1.0)

data/README.md

@@ -2,7 +2,7 @@
 
 Treyja is a Ruby script which takes `Tensors` protobin and outputs in human readable way.
 
-## Install
+## Installation
 
 Make sure you have `ruby` and `gem` commands installed.
 
@@ -13,34 +13,19 @@ $ gem --version
 2.7.6
 ```
 
-Using `specific_install` gem, you can install `treyja` gem directly from the directlyGitHub repo.
+Install it yourself as:
 
-The following command will do it:
-
-```console
-$ gem install specific_install
-$ gem specific_install https://github.com/xc-jp/treyja-ruby.git
-```
-
-You can check if `treyja` is properly installed by:
-
-```console
-% gem list treyja
-
-*** LOCAL GEMS ***
-
-treyja (0.1.0)
-```
+    $ gem install treyja
 
 
 ## How to use
 
-Assume that we are in the root dir of `alderaan` and the path to the project root of treyja-ruby is set to `$TREYJA_RUBY_ROOT`.
+Assume that we are in the root dir of `alderaan`.
 
 ### To show in JSON format
 
 ```console
-$ draupnir get draupnir/test/fixtures/configuration/literal.yaml | $TREYJA_RUBY_ROOT/bin/treyja json
+$ draupnir get draupnir/test/fixtures/configuration/literal.yaml | treyja json
 {"tensors":[{"dims":[3,2],"dataType":"FLOAT","int32Data":[],"int64Data":[],"uint32Data":[],"uint64Data":[],"floatData":[0.2,0.30000001,1,2,3,5],"doubleData":[]}]}
 {"tensors":[{"dims":[3,2],"dataType":"FLOAT","int32Data":[],"int64Data":[],"uint32Data":[],"uint64Data":[],"floatData":[0,0,0.5,0.60000002,0.69999999,0.80000001],"doubleData":[]}]}
 ```
@@ -50,7 +35,7 @@ $ draupnir get draupnir/test/fixtures/configuration/literal.yaml | $TREYJA_RUBY_
 [jq](https://stedolan.github.io/jq/) command is handy to pretty print JSON.
 
 ```console
-$ draupnir get draupnir/test/fixtures/configuration/literal.yaml | $TREYJA_RUBY_ROOT/bin/treyja json | jq
+$ draupnir get draupnir/test/fixtures/configuration/literal.yaml | treyja json | jq
 {
   "tensors": [
     {
@@ -111,10 +96,10 @@ Shape of input tensors must be 3 dimensional and channel count must be 1 (graysc
 Element type of input tensors must be uint8, int8, float or double.
 
 ```console
-$ draupnir get mnist --take 100 | $TREYJA_RUBY_ROOT/bin/treyja image --output tmp/mnist-images
+$ draupnir get mnist --take 100 | treyja image --output tmp/mnist-images
 ```
 
-![mnits-0](https://user-images.githubusercontent.com/1730718/48336250-3b13a100-e6a3-11e8-9f39-558ab792ac01.png)
+![mnist-0](https://user-images.githubusercontent.com/1730718/48336250-3b13a100-e6a3-11e8-9f39-558ab792ac01.png)
 ![mnist-1](https://user-images.githubusercontent.com/1730718/48352360-df134180-e6cf-11e8-8de0-7d083315a55d.png)
 ![mnist-2](https://user-images.githubusercontent.com/1730718/48352418-f8b48900-e6cf-11e8-8bf5-0714c152911c.png)
 ![mnist-3](https://user-images.githubusercontent.com/1730718/48352488-21d51980-e6d0-11e8-9af7-907441dfe540.png)

data/exe/treyja

@@ -1,211 +1,5 @@
 #! /usr/bin/env ruby
 
-$LOAD_PATH << File.join(__dir__, "../lib")
-require 'tensors_pb.rb'
+require "treyja/exe"
 
-$unabbrev = {
-  'h' => 'help',
-  'help' => 'help',
-  'output' => 'output',
-  'normalize' => 'normalize',
-  'round' => 'round'
-}
-
-$options = [
-  *ARGV.map { |k| (ms = k.match(/^--?(\w+)/)) ? [$unabbrev[ms[1]], true] : nil },
-  *ARGV.each_cons(2).map { |k, v| (v !~ /^-/ && ms = k.match(/^--?(\w+)/)) ? [$unabbrev[ms[1]], v] : nil }
-  ].compact.to_h
-
-$command, $file = ARGV.take_while { |s| s !~ /^-/ }
-if $command.nil? || $options["help"]
-  puts <<EOS
-Usage: treyja COMMAND [FILE] [OPTIONS]
-
-  If FILE is not given, it reads from STDIN.
-
-Available options:
-  -h,--help                Show this help text
-  --output DIR             Directory to output images
-  --normalize              Normalize floating values (image only)
-  --round INT              Digit to round floating values (default: 4)
-
-Available commands:
-  json                     Output tensors in JSON format
-  csv                      Output tensors in CSV format
-  image                    Create png images and output to DIR
-  dump                     Dump in the format of inner expression
-EOS
-  exit
-end
-
-$round = $options.fetch('round', 4).to_f
-if $round > 0
-  class Float
-    def to_s_with_round
-      self.round($round).to_s_without_round
-    end
-    alias_method :to_s_without_round, :to_s
-    alias_method :to_s, :to_s_with_round
-  end
-
-  class Double
-    def to_s_with_round
-      self.round($round).to_s_without_round
-    end
-    alias_method :to_s_without_round, :to_s
-    alias_method :to_s, :to_s_with_round
-  end
-end
-
-def reader
-  io = $file ? open($file) : STDIN
-  io.binmode
-
-  Enumerator.new do |y|
-    while magic = io.read(4)
-      raise "Incorrect magic bytes" unless magic == 'XCIX'
-      length = io.read(8).reverse.unpack("Q").first.to_i
-      y << Tensors::TensorsProto.decode(io.read(length))
-    end
-  end
-end
-
-
-def write_image tensor, k, file
-  width, height, channel = tensor.dims
-  width ||= 1
-  height ||= 1
-  channel ||= 1
-  offset = width * height * channel * k
-  range = offset ... (width * height * channel + offset)
-
-  normalizer =
-    if $options['normalize']
-      -> (xs) {
-        min, max = xs.minmax
-        xs.map { |f| ((f - min) / (max - min) * 255).to_i }
-      }
-    else
-      -> (xs) { xs.map { |f| (f * 255).to_i } }
-    end
-
-  depth, org_data =
-    case tensor.data_type
-    when :UINT8
-      [8, tensor.byte_data.unpack('C*')[range]]
-    when :INT8
-      [8, tensor.byte_data.unpack('C*')[range]]
-    when :FLOAT
-      [8, normalizer.call(tensor.float_data[range])]
-    when :DOUBLE
-      [8, normalizer.call(tensor.double_data[range])]
-    else
-      raise "unsupported data type: #{tensor.data_type}"
-    end
-
-  color_type =
-    case channel
-    when 1
-      0                         # grayscale
-    when 2
-      4                         # grayscale and alpha
-    when 3
-      2                         # rgb
-    when 4
-      6                         # rgba
-    else
-      raise "unsupported channel: #{channel}"
-    end
-
-  raw_data = (0...height).map do |y|
-    (0...width).map do |x|
-      (0...channel).map do |ch|
-        org_data[ch * height * width + y * width + x]
-      end
-    end
-  end
-
-  def chunk(type, data)
-    [data.bytesize, type, data, Zlib.crc32(type + data)].pack("NA4A*N")
-  end
-
-  open(file, 'w') do |io|
-    io.print "\x89PNG\r\n\x1a\n"
-    io.print chunk("IHDR", [width, height, depth, color_type, 0, 0, 0].pack("NNCCCCC"))
-    img_data = raw_data.map {|line| ([0] + line.flatten).pack("C*") }.join
-    io.print chunk("IDAT", Zlib::Deflate.deflate(img_data))
-    io.print chunk("IEND", "")
-  end
-end
-
-def dims_to_indices dims
-  if dims.empty?
-    [[]]
-  else
-    x = dims.first
-    xs = dims[1..-1]
-    Enumerator.new do |y|
-      dims_to_indices(xs).each do |is|
-        x.times.each do |i|
-          y << [i, *is]
-        end
-      end
-    end
-  end
-end
-
-def raw_data tensor
-  case tensor.data_type
-  when :INT8, :UINT8
-    tensor.byte_data.unpack('C*')
-  when :FLOAT
-    tensor.float_data
-  when :DOUBLE
-    tensor.double_data
-  else
-    raise "unsupported data type: #{tensor.data_type}"
-  end
-end
-
-case $command
-when "dump"
-  reader.each do |ts|
-    p ts
-    STDOUT.flush
-  end
-when "json"
-  reader.each do |ts|
-    puts ts.to_json
-    STDOUT.flush
-  end
-when "image"
-  dir = $options["output"]
-  raise "--output option required" unless dir
-
-  FileUtils.mkdir_p dir
-
-  require "zlib"
-  reader.each_with_index do |tensors, i|
-    tensors.tensors.each_with_index do |tensor, j|
-      n = tensor.dims.drop(3).inject(1, :*) # Drop width, height and channel and fold the rest
-      n.times.each do |k|
-        postfix = ([i, j] + (n > 1 ? [k] : [])).map(&:to_s).join('-')
-        dst = File.join(dir, "image-#{postfix}.png")
-        write_image tensor, k, dst
-      end
-    end
-  end
-when "csv"
-  if head = reader.first
-    headers = head.tensors.each_with_index.flat_map do |t, i|
-      dims_to_indices(t.dims).map do |ix|
-        [i.to_s, ix.empty? ? nil : ix.reverse.map(&:to_s).join('_')].compact.join('-')
-      end
-    end
-    puts headers.join(',')
-    puts head.tensors.flat_map { |t| raw_data(t) }.map(&:to_s).join(',')
-    reader.each do |ts|
-      puts ts.tensors.flat_map { |t| raw_data(t) }.map(&:to_s).join(',')
-    end
-  end
-end
+Treyja::Exe.new(ARGV).run

data/lib/treyja/command/csv.rb

@@ -0,0 +1,57 @@
+module Treyja
+  module Command
+    class Csv
+      attr_reader :reader
+
+      def initialize reader
+        @reader = reader
+      end
+
+      def run
+        if head = reader.first
+          headers = head.tensors.each_with_index.flat_map do |t, i|
+            dims_to_indices(t.dims).map do |ix|
+              [i.to_s, ix.empty? ? nil : ix.reverse.map(&:to_s).join('_')].compact.join('-')
+            end
+          end
+          puts headers.join(',')
+          puts head.tensors.flat_map { |t| raw_data(t) }.map(&:to_s).join(',')
+          reader.each do |ts|
+            puts ts.tensors.flat_map { |t| raw_data(t) }.map(&:to_s).join(',')
+          end
+        end
+      end
+
+      private
+
+      def dims_to_indices dims
+        if dims.empty?
+          [[]]
+        else
+          x = dims.first
+          xs = dims[1..-1]
+          Enumerator.new do |y|
+            dims_to_indices(xs).each do |is|
+              x.times.each do |i|
+                y << [i, *is]
+              end
+            end
+          end
+        end
+      end
+
+      def raw_data tensor
+        case tensor.data_type
+        when :INT8, :UINT8
+          tensor.byte_data.unpack('C*')
+        when :FLOAT
+          tensor.float_data
+        when :DOUBLE
+          tensor.double_data
+        else
+          raise "unsupported data type: #{tensor.data_type}"
+        end
+      end
+    end
+  end
+end

data/lib/treyja/command/dump.rb

@@ -0,0 +1,18 @@
+module Treyja
+  module Command
+    class Dump
+      attr_reader :reader
+
+      def initialize reader
+        @reader = reader
+      end
+
+      def run
+        reader.each do |ts|
+          p ts
+          STDOUT.flush
+        end
+      end
+    end
+  end
+end

data/lib/treyja/command/help.rb

@@ -0,0 +1,25 @@
+module Treyja
+  module Command
+    class Help
+      def run
+        puts <<EOS
+Usage: treyja COMMAND [FILE] [OPTIONS]
+
+  If FILE is not given, it reads from STDIN.
+
+Available options:
+  -h,--help                Show this help text
+  --output DIR             Directory to output images
+  --normalize              Normalize floating values (image only)
+  --round INT              Digit to round floating values (default: 4)
+
+Available commands:
+  json                     Output tensors in JSON format
+  csv                      Output tensors in CSV format
+  image                    Create png images and output to DIR
+  dump                     Dump in the format of inner expression
+EOS
+      end
+    end
+  end
+end

data/lib/treyja/command/image.rb

@@ -0,0 +1,104 @@
+require "zlib"
+
+module Treyja
+  module Command
+    class Image
+      attr_reader :reader, :output_dir, :options
+
+      def initialize reader, output_dir, options = {}
+        @reader = reader
+        @output_dir = output_dir
+        @options = options
+      end
+
+      def run
+        FileUtils.mkdir_p output_dir
+
+        reader.each_with_index do |tensors, i|
+          tensors.tensors.each_with_index do |tensor, j|
+            n = tensor.dims.drop(3).inject(1, :*) # Drop width, height and channel and fold the rest
+            n.times.each do |k|
+              postfix = ([i, j] + (n > 1 ? [k] : [])).map(&:to_s).join('-')
+              dst = File.join(output_dir, "image-#{postfix}.png")
+              write_image tensor, k, dst
+            end
+          end
+        end
+        reader.each do |ts|
+          p ts
+          STDOUT.flush
+        end
+      end
+
+      private
+
+      def write_image tensor, k, file
+        width, height, channel = tensor.dims
+        width ||= 1
+        height ||= 1
+        channel ||= 1
+        offset = width * height * channel * k
+        range = offset ... (width * height * channel + offset)
+
+        normalizer =
+          if options['normalize']
+            -> (xs) {
+              min, max = xs.minmax
+              xs.map { |f| ((f - min) / (max - min) * 255).to_i }
+            }
+          else
+            -> (xs) { xs.map { |f| (f * 255).to_i } }
+          end
+
+        depth, org_data =
+               case tensor.data_type
+               when :UINT8
+                 [8, tensor.byte_data.unpack('C*')[range]]
+               when :INT8
+                 [8, tensor.byte_data.unpack('C*')[range]]
+               when :FLOAT
+                 [8, normalizer.call(tensor.float_data[range])]
+               when :DOUBLE
+                 [8, normalizer.call(tensor.double_data[range])]
+               else
+                 raise "unsupported data type: #{tensor.data_type}"
+               end
+
+        color_type =
+          case channel
+          when 1
+            0                         # grayscale
+          when 2
+            4                         # grayscale and alpha
+          when 3
+            2                         # rgb
+          when 4
+            6                         # rgba
+          else
+            raise "unsupported channel: #{channel}"
+          end
+
+        raw_data = (0...height).map do |y|
+          (0...width).map do |x|
+            (0...channel).map do |ch|
+              org_data[ch * height * width + y * width + x]
+            end
+          end
+        end
+
+        def chunk(type, data)
+          [data.bytesize, type, data, Zlib.crc32(type + data)].pack("NA4A*N")
+        end
+
+        open(file, 'w') do |io|
+          io.print "\x89PNG\r\n\x1a\n"
+          io.print chunk("IHDR", [width, height, depth, color_type, 0, 0, 0].pack("NNCCCCC"))
+          img_data = raw_data.map {|line| ([0] + line.flatten).pack("C*") }.join
+          io.print chunk("IDAT", Zlib::Deflate.deflate(img_data))
+          io.print chunk("IEND", "")
+        end
+      end
+
+    end
+  end
+end

data/lib/treyja/command/json.rb

@@ -0,0 +1,18 @@
+module Treyja
+  module Command
+    class Json
+      attr_reader :reader
+
+      def initialize reader
+        @reader = reader
+      end
+
+      def run
+        reader.each do |ts|
+          puts ts.to_json
+          STDOUT.flush
+        end
+      end
+    end
+  end
+end

data/lib/treyja/exe.rb

@@ -0,0 +1,81 @@
+require "optparse"
+
+require "treyja/reader"
+require "treyja/command/csv"
+require "treyja/command/dump"
+require "treyja/command/help"
+require "treyja/command/image"
+require "treyja/command/json"
+
+class Float
+  def self.round_digits
+    @round_digits
+  end
+
+  def to_s_with_round
+    Float.round_digits ?
+      self.round(Float.round_digits).to_s_without_round :
+      self.to_s_without_round
+  end
+
+  alias_method :to_s_without_round, :to_s
+  alias_method :to_s, :to_s_with_round
+end
+
+module Treyja
+  class Exe
+    def initialize args
+      @args = args
+    end
+
+    def options
+      @options ||= {}
+    end
+
+    def patch_round round
+      if round > 0
+        ::Float.instance_variable_set "@round_digits", round
+      end
+    end
+
+    def run
+      opts = OptionParser.new
+      opts.on("--output DIR")
+      opts.on("--normalize")
+      opts.on("--round INT")
+      opts.on("--version") do
+        puts "Version: #{Treyja::VERSION}"
+        return
+      end
+      opts.on("-h", "--help") do
+        Treyja::Command::Help.new.run
+        return
+      end
+      command, file = opts.parse!(@args, into: options)
+
+      patch_round options.fetch(:round, 4).to_f
+      case command
+      when "dump"
+        reader = Treyja::Reader.new file
+        Treyja::Command::Dump.new(reader).run
+      when "json"
+        reader = Treyja::Reader.new file
+        Treyja::Command::Json.new(reader).run
+      when "image"
+        output_dir = options[:output]
+        raise "--output option required" unless output_dir
+
+        reader = Treyja::Reader.new file
+        Treyja::Command::Image.new(reader, output_dir, options.slice(:normalize)).run
+      when "csv"
+        reader = Treyja::Reader.new file
+        Treyja::Command::Csv.new(reader).run
+      when nil
+        Treyja::Command::Help.new.run
+      else
+        puts "Unknown command: #{command}"
+        Treyja::Command::Help.new.run
+      end
+    end
+  end
+end

data/lib/{tensors_pb.rb → treyja/proto/tensors_pb.rb}

@@ -3,7 +3,7 @@
 
 require 'google/protobuf'
 
-require 'tensor_pb'
+require 'treyja/proto/tensor_pb'
 Google::Protobuf::DescriptorPool.generated_pool.build do
   add_message "tensors.TensorsProto" do
     repeated :tensors, :message, 1, "tensor.TensorProto"

data/lib/treyja/reader.rb

@@ -0,0 +1,25 @@
+require "treyja/proto/tensors_pb"
+
+module Treyja
+  class Reader
+    extend Forwardable
+    include Enumerable
+
+    delegate [:each] => :@enumerator
+
+    MAGIC_BYTES = "XCIX"
+
+    def initialize file = nil
+      io = file ? open(file) : STDIN
+      io.binmode
+
+      @enumerator = Enumerator.new do |y|
+        while magic = io.read(4)
+          raise "Incorrect magic bytes" unless magic == MAGIC_BYTES
+          length = io.read(8).reverse.unpack("Q").first.to_i
+          y << Tensors::TensorsProto.decode(io.read(length))
+        end
+      end
+    end
+  end
+end

data/lib/treyja/version.rb

@@ -1,3 +1,3 @@
 module Treyja
-  VERSION = "0.1.2"
+  VERSION = "0.1.3"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: treyja
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - Cross Compass
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-04-26 00:00:00.000000000 Z
+date: 2019-05-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: google-protobuf
@@ -102,12 +102,18 @@ files:
 - default.nix
 - exe/treyja
 - gemset.nix
-- lib/tensor_pb.rb
-- lib/tensors_pb.rb
 - lib/treyja.rb
+- lib/treyja/command/csv.rb
+- lib/treyja/command/dump.rb
+- lib/treyja/command/help.rb
+- lib/treyja/command/image.rb
+- lib/treyja/command/json.rb
+- lib/treyja/exe.rb
+- lib/treyja/proto/tensor_pb.rb
+- lib/treyja/proto/tensors_pb.rb
+- lib/treyja/reader.rb
 - lib/treyja/version.rb
 - treyja.gemspec
-- treyja/.gitignore
 homepage: http://github.com/xc-jp/treyja-ruby
 licenses:
 - MIT

data/treyja/.gitignore

@@ -1,11 +0,0 @@
-/.bundle/
-/.yardoc
-/_yardoc/
-/coverage/
-/doc/
-/pkg/
-/spec/reports/
-/tmp/
-
-# rspec failure tracking
-.rspec_status