CooCoo 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 98b06c828793e0ecc63255ac42b21b1860bb392d
+  data.tar.gz: d280b835315f6de205602b40150c6d68e638a42d
+SHA512:
+  metadata.gz: 265aef5a22270f7476fae2327e9f0f0d03817a74026c85701ecfb444e468a6ff475b0f58f9ecdaeb03f802d041d29d63716d7e4df9e6695c044099c01d966683
+  data.tar.gz: 595de49ff692e78f5b49ddcf01ae510fde8f0227253d00f3b67f9fe6b1a5b2ba533589f14035d006b363bc8b50f39c2e58e6ff6c6c0c1b38345bf8104a9ed693

data/.gitignore

@@ -0,0 +1,16 @@
+*~
+\#*#
+*.bak
+*.neural[-_]model
+*.coocoo[-_]model
+*.o
+*.so
+*.exe
+*.dll
+doc/spec
+doc/spec.html
+doc/coverage
+ext/buffer/*.cu
+!ext/buffer/*.pre.cu
+ext/buffer/*ffi.rb
+ext/buffer/*.h

data/CooCoo.gemspec

@@ -0,0 +1,47 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'coo-coo/version'
+
+Gem::Specification.new do |s|
+  s.name        = "CooCoo"
+  s.version     = CooCoo::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Nolan Eakins"]
+  s.email       = ["sneakin+coocoo@semanticgap.com"]
+  s.homepage    = "https://CooCoo.network/"
+  s.summary     = "Neural networks in Ruby and CUDA."
+  s.license     = "GPL"
+
+  if s.respond_to?(:metadata)
+    s.metadata['yard.run'] = 'yri'
+  else
+    raise "RubyGems 2.0 or newer is required to protect against " \
+      "public gem pushes."
+  end
+
+  s.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  s.bindir        = "bin"
+  s.executables   = s.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_development_dependency "bundler", "~> 1.14"
+  s.add_development_dependency "rake", "~> 10.0"
+  s.add_development_dependency "rspec", "~> 3.0"
+  s.add_development_dependency "yard"
+  s.add_development_dependency "yard-rspec"
+  s.add_development_dependency "pry", "~> 0.11.3"
+  s.add_development_dependency 'simplecov'
+  s.add_development_dependency 'coderay'
+
+  s.add_dependency 'nmatrix'
+  s.add_dependency 'parallel'
+  s.add_dependency 'nokogiri'
+  s.add_dependency 'ruby-progressbar'
+  s.add_dependency 'chunky_png'
+  s.add_dependency 'cairo'
+  s.add_dependency 'colorize'
+  s.add_dependency 'ffi'
+end

data/Gemfile

@@ -0,0 +1,4 @@
+source :rubygems
+gemspec
+
+#gem 'simplecov', :require => false, :group => :test

data/Gemfile.lock

@@ -0,0 +1,88 @@
+PATH
+  remote: .
+  specs:
+    CooCoo (0.1.0)
+      cairo
+      chunky_png
+      colorize
+      ffi
+      nmatrix
+      nokogiri
+      parallel
+      ruby-progressbar
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    backports (3.10.3)
+    cairo (1.15.11)
+      native-package-installer (>= 1.0.3)
+      pkg-config (>= 1.2.2)
+    cairo (1.15.11-x64-mingw32)
+      native-package-installer (>= 1.0.3)
+      pkg-config (>= 1.2.2)
+    chunky_png (1.3.8)
+    coderay (1.1.2)
+    colorize (0.8.1)
+    diff-lcs (1.3)
+    docile (1.1.5)
+    ffi (1.9.18)
+    ffi (1.9.18-x64-mingw32)
+    json (2.1.0)
+    method_source (0.9.0)
+    mini_portile2 (2.3.0)
+    native-package-installer (1.0.6)
+    nmatrix (0.2.4)
+      packable (~> 1.3, >= 1.3.5)
+    nokogiri (1.8.1)
+      mini_portile2 (~> 2.3.0)
+    nokogiri (1.8.1-x64-mingw32)
+      mini_portile2 (~> 2.3.0)
+    packable (1.3.9)
+      backports
+    parallel (1.12.1)
+    pkg-config (1.2.8)
+    pry (0.11.3)
+      coderay (~> 1.1.0)
+      method_source (~> 0.9.0)
+    rake (10.5.0)
+    rspec (3.7.0)
+      rspec-core (~> 3.7.0)
+      rspec-expectations (~> 3.7.0)
+      rspec-mocks (~> 3.7.0)
+    rspec-core (3.7.0)
+      rspec-support (~> 3.7.0)
+    rspec-expectations (3.7.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.7.0)
+    rspec-mocks (3.7.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.7.0)
+    rspec-support (3.7.0)
+    ruby-progressbar (1.9.0)
+    simplecov (0.15.1)
+      docile (~> 1.1.0)
+      json (>= 1.8, < 3)
+      simplecov-html (~> 0.10.0)
+    simplecov-html (0.10.2)
+    yard (0.9.12)
+    yard-rspec (0.1)
+      yard
+
+PLATFORMS
+  ruby
+  x64-mingw32
+
+DEPENDENCIES
+  CooCoo!
+  bundler (~> 1.14)
+  coderay
+  pry (~> 0.11.3)
+  rake (~> 10.0)
+  rspec (~> 3.0)
+  simplecov
+  yard
+  yard-rspec
+
+BUNDLED WITH
+   1.15.4

data/README.md

@@ -0,0 +1,123 @@
+CooCoo
+==========
+
+Copyright © 2017-2018 [Nolan Eakins](mailto:sneakin+at+semanticgap.com)
+
+THIS IS NOT PRODUCTION QUALITY. USE AT YOUR OWN RISK.
+
+[Home page](https://coocoo.network/)
+[GitHub](https://github.com/sneakin/CooCoo/)
+
+
+Description
+----------
+
+A neural network library implemented in Ruby with a CUDA backend.
+
+
+Dependencies
+-------------
+
+* [NVIDIA's CUDA Toolkit](https://developer.nvidia.com/cuda-downloads)
+* Any one of the following C/C++ environments:
+  * [GCC](https://gcc.gnu.org/)
+  * [MSYS2](http://www.msys2.org/)
+* [Ruby](https://www.ruby-lang.org/): install via your distribution package manager or MSYS2's `pacman -S ruby`
+* [Bundler](http://bundler.io/)
+* [Cairo](https://www.cairographics.org/rcairo/)
+
+
+Usage
+----------
+
+### Coming soon
+
+    $ gem install CooCoo
+
+
+### Install
+
+First the required dependencies need to be installed. The dependencies include the CUDA compiler and RubyGems.
+
+Once the CUDA toolkit is installed, make sure it and Visual Studio's C++ compiler are in your path. Under MSYS2, use something like:
+
+    $ export PATH=/c/Program\ Files\ \(x86\)/Microsoft\ Visual\ Studio/2017/Community/VC/Tools/MSVC/14.10.25017/bin/HostX64/x64:$PATH
+    $ export PATH=/c/Program\ Files/NVIDIA\ GPU\ Computing\ Toolkit/CUDA/v9.0/bin:$PATH
+
+RubyGems are installed with Bundler: `bundle install`
+
+And the extension is built with: `rake compile`
+
+Then to run an example: `bundle exec ruby -Ilib -Iexamples examples/seeds.rb`
+
+Or IRB: `bundle exec irb -Ilib`
+
+Or IRB: `bundle exec ruby -Ilib`
+
+
+### Code
+
+```ruby
+require 'coo-coo'
+
+network = CooCoo::Network.new()
+# create the layers
+network.layer(CooCoo::Layer.new(28 * 28, 100))
+network.layer(CooCoo::Layer.new(100, 10))
+
+# learn
+training_data = [ [expected_output, input_data_array ], ...]
+
+trainer = CooCoo::Trainer::Stochastic.new
+trainer.train(network: network,
+              data: training_data,
+              learning_rate: learning_rate,
+              batch_size: batch_size) do |stats|
+  # called every batch_size
+  puts("Batch #{batch} took #{stats.total_time} seconds with an average loss of #{stats.average_loss}.")
+end
+
+# store to disk
+network.save("my_first_network.coo-coo_model")
+
+# load from disk
+loaded_net = CooCoo::Network.load!("my_first_network.coo-coo_model")
+
+# predict
+output = loaded_network.predict([ 0, 0, 0, ... ])
+# => [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ]
+
+```
+
+Examples
+----------
+
+All examples use `OptParse`, so refer to the output of running the example with `--help`. With no arguments, typically a network will be generated, trained, and tested without saving.
+
+To run an example: `bundle exec ruby -Ilib -Iexamples examples/EXAMPLE.rb --help`
+
+### [char-rnn](examples/char-rnn.rb)
+
+A recursive network that learns byte sequences.
+
+### [UCI Wheat Seed Classifier](examples/seeds.rb)
+
+Inspired by the IBAFSIP, this uses the UCI Machine Learning Repository's [wheat seed dataset](http://archive.ics.uci.edu/ml/datasets/seeds) to predict the type of seed given seven parameters.
+
+### [MNIST Classifier](examples/mnist_classifier.rb)
+
+[The MNIST Database](http://yann.lecun.com/exdb/mnist/) is used to train a 10 digit classifier.
+
+### [MNIST Dream](examples/mnist_dream.rb)
+
+Uses a network trained with the MNIST Classifier to generate valid inputs via backpropagation.
+
+
+Credits
+----------
+
+Loosely based on [Implement Backpropagation Algorithm From Scratch in Python](http://machinelearningmastery.com/implement-backpropagation-algorithm-scratch-python/)
+
+And more than a few of [Siraj Raval's videos](https://www.youtube.com/channel/UCWN3xxRkmTPmbKwht9FuE5A)
+
+Cross entropy & Softmax sorted out with [DeepNotes.io](https://deepnotes.io/softmax-crossentropy)

data/Rakefile

@@ -0,0 +1,81 @@
+require 'bundler/gem_tasks'
+
+task :default => [ 'spec:coverage', 'doc' ]
+
+desc "Clean any output files."
+task :clean => :clean_ext do
+  sh("rm -rf doc/coverage doc/rdoc doc/spec.html doc/api")
+end
+
+task :clean_ext do
+  Dir.glob("ext/*").each do |entry|
+    sh("cd #{entry}; rake clean") if File.exists?(File.join(entry, "Rakefile"))
+  end
+end
+
+desc "Compile the extensions."
+if ENV["USE_CUDA"] != "0"
+  task :compile => "ext/buffer/Rakefile" do |t|
+    pwd = Dir.pwd
+    Dir.chdir(File.dirname(t.source))
+    sh("rake")
+    Dir.chdir(pwd)
+  end
+else
+  task :compile do
+  end
+end
+
+desc 'Create the YARDocs'
+require 'yard'
+YARD::Rake::YardocTask.new(:doc) do |t|
+  t.files = [ 'lib/**/*.rb', 'examples/**/*.rb', 'spec/**/*.spec', '-', 'README.md' ]
+  t.options = [ '--plugin', 'rspec', '-o', 'doc/api' ]
+end
+
+desc "Run the rspecs."
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec => :compile) do |t, args|
+  t.pattern = "spec/**/*.spec"
+end
+
+namespace :spec do
+  desc "Run the specs with HTML output."
+  RSpec::Core::RakeTask.new(:html => :compile) do |t, args|
+    t.pattern = "spec/**/*.spec"
+    t.rspec_opts = '-fhtml -o doc/spec.html'
+  end
+
+  desc "Run the specs with code coverage."
+  task :coverage => :compile do
+    ENV['COVERAGE'] = 'true'
+    Rake::Task['spec:html'].execute
+  end
+end
+
+desc "Run Ruby with everything in the search paths."
+task :run => :compile do
+  args = $*[1, $*.size - 1]
+  exec("bundle exec ruby -Ilib -Iexamples #{args.join(' ')}")
+end
+
+desc "Start an IRB session with everything loaded."
+task :shell => :compile do
+  exec("bundle exec irb -Ilib -Iexamples -rcoo-coo/shell")
+end
+
+desc "Start a Pry session with everything loaded."
+task :pry => :compile do
+  exec("bundle exec pry -Ilib -Iexamples -rcoo-coo/shell")
+end
+
+namespace :www do
+  desc "Upload the website"
+  task :upload do
+    user = ENV.fetch("COOCOO_USER")
+    sh("ssh #{user}@coocoo.network mkdir -p \\~/www/coocoo.network/public/images")
+    sh("scp www/index.html #{user}@coocoo.network:~/www/coocoo.network/public/index.html")
+    sh("scp www/images/screamer.png #{user}@coocoo.network:~/www/coocoo.network/public/images/screamer.png")
+  end
+end
+

data/bin/cuda-dev-info

@@ -0,0 +1,25 @@
+#!/bin/env ruby
+
+$: << File.join(File.dirname(__FILE__), '..', 'lib')
+require 'coo-coo/cuda'
+
+unless CooCoo::CUDA.available?
+  puts("Cuda not available")
+  exit(-1)
+end
+
+puts("Device\t#{CooCoo::CUDA::Runtime.get_device} / #{CooCoo::CUDA::Runtime.device_count}")
+props = CooCoo::CUDA::Runtime::DeviceProperties.new
+err = CooCoo::CUDA::Runtime.cudaGetDeviceProperties(props, 0)
+raise CUDAError(err) if err != 0
+
+props.members.each do |m|
+  value = props[m]
+  if m != :name && value.kind_of?(FFI::Struct::InlineArray)
+    value.each_with_index do |v, i|
+      puts("#{m}[#{i}]\t#{v}")
+    end
+  else
+    puts("#{m}\t#{value}")
+  end
+end

data/bin/cuda-free

@@ -0,0 +1,28 @@
+#!/bin/env ruby
+
+$: << File.join(File.dirname(__FILE__), '..', 'lib')
+require 'coo-coo/cuda'
+
+require 'ostruct'
+options = OpenStruct.new
+options.follow = ARGV[0] == '-f'
+options.timeout = (ARGV[1] || 1).to_f
+
+unless options.follow
+  puts("Device\tBytes Free\tBytes Total\tPercent")
+end
+
+follow = true
+$stdout.sync = true
+
+while follow
+  follow = options.follow
+  CooCoo::CUDA::Runtime.device_count.times do |device|
+    CooCoo::CUDA::Runtime.set_device(device)
+    free, total = CooCoo::CUDA.memory_info
+    $stdout.puts("%i\t%i\t%i\t%.3f" % [ device, free, total, (100.0 * free / total.to_f) ])
+  end
+
+  $stdout.flush
+  sleep(options.timeout)
+end

data/bin/cuda-free-trend

@@ -0,0 +1,7 @@
+#!/bin/zsh
+
+./bin/cuda-free -f 0.25 |
+    stdbuf -o0 cut -f 4 |
+    trend -t "${TREND_TITLE:-Free CUDA Memory}" -c 1a -geometry ${TREND_GEOMETRY:-640x64-0-24} - ${TREND_SIZE:-$((4*60*4*5))x2} 0 100
+
+[[ -z `jobs -p` ]] || kill $(jobs -p)

data/bin/ffi-gen

@@ -0,0 +1,267 @@
+#!/bin/env ruby
+
+require 'ffi'
+
+class ExportScanner
+  NEEDLE = "PUBLIC"
+  NVCCFLAGS = []
+
+  attr_reader :structs
+  attr_reader :functions
+
+  def initialize(typedefs = Hash.new)
+    @functions = Array.new
+    @structs = Array.new
+    @typedefs = typedefs
+  end
+
+  FunctionPrototype = Struct.new(:name, :return_type, :args, :definition)
+  
+  class Argument
+    attr_accessor :const
+    attr_accessor :type
+    attr_accessor :name
+    attr_accessor :pointer
+    
+    def initialize(str)
+      m = str.match(/\s*(((\w+) )+\s*(([*]+)?(\w+)?)\s*)/m)
+      if m
+        self.name = m[-1]
+        self.const = m[0] =~ /const/
+        self.pointer = m[-2]
+        self.type = m[3]
+      end
+    end
+  end
+
+  def parse_args(arg_str)
+    args = arg_str.split(',').collect(&:strip)
+    arg_types = args.collect do |a|
+      Argument.new(a)
+    end
+  end
+
+  def scan_function(f)
+    m = f.match(/(\w+)\s+(\w+)\s*\((.*)\)/)
+    if m
+      func = m[2]
+      args = parse_args(m[3])
+      ret = m[1].strip
+      FunctionPrototype.new(func, ret, args, f)
+    end
+  end
+
+  def scan_for_functions(path)
+    `nvcc -E -DIN_PUBLIC=1 -D#{NEEDLE}=#{NEEDLE} #{NVCCFLAGS.join(' ')} #{path} | grep -e '#{NEEDLE}'`.
+      split("\n").
+      collect { |f| m = f.match(/(#{NEEDLE}.*[)]([^;]|$))/); m && m[1] }.
+      reject(&:nil?).
+      collect { |f| scan_function(f[0, f.index(')') + 1]) }
+  end
+
+  StructDef = Struct.new(:name, :definition, :fields)
+  StructField = Struct.new(:name, :definition, :type, :pointer)
+
+  def scan_struct_field(field)
+    m = field.match(/\s*(((\w+) )+\s*(([*]+)?(\w+))\s*)/m)
+    if m
+      StructField.new(m[-1], field, m[2].strip, m[-2])
+    end
+  end
+  
+  def scan_struct_fields(fields)
+    fields.collect { |field| scan_struct_field(field) }.reject(&:nil?)
+  end
+
+  def scan_for_structs_inner(data, acc = [])
+    m = data.match(/(typedef\s+struct\s+(\w+)\s+{(.*)}\s+[*](\w+);)(.*)/m)
+    if m
+      acc << StructDef.new(m[4], m[1], scan_struct_fields(m[3].split(';').collect(&:strip)))
+      scan_for_structs_inner(m[5], acc)
+    else
+      acc
+    end
+  end
+  
+  def scan_for_structs(path)
+    scan_for_structs_inner(File.read(path))
+  end
+
+  def scan(path)
+    @functions += scan_for_functions(path)
+    @structs += scan_for_structs(path)
+
+    self
+  end
+
+  def header(name)
+    head = File.basename(name).upcase
+    <<-EOT
+\#ifndef #{head}
+\#define #{head}
+
+\#include "public.h"
+
+extern "C" {
+  #{structs.collect(&:definition).join('\n')}
+
+  #{functions.collect(&:definition).join(";\n  ")};
+}
+
+\#endif /* #{head} */
+EOT
+  end
+
+  def ffi_type(a, pointer = nil)
+    a = @typedefs[a] if @typedefs.has_key?(a)
+    if FFI::TypeDefs[a.to_sym]
+      ":" + FFI::TypeDefs.key(FFI::TypeDefs[a.to_sym]).to_s
+    elsif pointer
+      "#{a}.auto_ptr"
+    else
+      a
+    end
+  end
+  
+  def ffi_def(f)
+    func = f.name
+    args = f.args.collect do |a|
+      if a.pointer
+        ":pointer"
+      else
+        ffi_type(a.type)
+      end
+    end
+
+    ret = f.return_type
+    ret = ffi_type(ret, true)
+
+    "    attach_function :#{func}, [ #{args.join(', ')} ], #{ret}"
+  end
+
+  def ffi_struct(s)
+    layout = s.fields.collect { |f| [ ":#{f.name}", ffi_type(f.type, f.pointer) ] }
+    <<-EOT
+class #{s.name} < ::FFI::Struct
+  layout(#{layout.join(', ')})
+
+  def self.release(ptr)
+    #{s.name.downcase}_free(ptr)
+  end
+end
+EOT
+  end
+  
+  def ffi(name, library)
+    <<-EOT
+module #{name}
+  module FFI
+    extend ::FFI::Library
+    ffi_lib Pathname.new(__FILE__).join('..', "#{library}.#{RbConfig::CONFIG['DLEXT']}").to_s
+
+#{structs.collect { |s| ffi_struct(s) }.compact.join("\n")}
+#{functions.collect { |f| ffi_def(f) }.compact.join("\n")}
+  end
+end
+EOT
+  end
+end
+
+def write_header(header, sources)
+  # empty the file for preprocessing
+  if header != nil && header != '-'
+    File.open(header, 'w') do |f|
+      f.puts
+    end
+  end
+
+  scanner = ExportScanner.new
+  sources.each(&-> (s) { scanner.scan(s) })
+
+  if header == nil || header == '-'
+    $stdout.puts(scanner.header("STDOUT"))
+  else
+    File.open(header, 'w') do |f|
+      f.puts(scanner.header(header.gsub('.', '_')))
+    end
+  end
+end
+
+def write_ffi(mod, library, output, sources, typedefs)
+  raise ArgumentError.new('No --module name given.') if mod == nil
+  raise ArgumentError.new('No --library given.') if library == nil
+
+  scanner = ExportScanner.new(typedefs)
+  sources.each(&->(s) { scanner.scan(s) })
+
+  if output == nil || output == '-'
+    $stdout.puts(scanner.ffi(mod, library))
+  else
+    File.open(output, 'w') do |f|
+      f.puts(scanner.ffi(mod, library))
+    end
+  end
+end
+
+require 'fileutils'
+def write_source(output, source)
+  FileUtils.copy(source, output)
+end
+
+
+if __FILE__ == $0
+  require 'ostruct'
+  options = OpenStruct.new
+  options.mode = :help
+  options.sources = Array.new
+  options.typedefs = Hash.new
+  
+  require 'optparse'
+  opts = OptionParser.new() do |o|
+    o.on('-h', 'Help') do
+      options.mode = :help
+    end
+
+    o.on('--header', 'Generate a C/C++ header.') do
+      options.mode = :header
+    end
+
+    o.on('--ffi', 'Generate a Ruby FFI.') do
+      options.mode = :ffi
+    end
+
+    o.on('--source', 'Generate C/C++ source code.') do
+      options.mode = :source
+    end
+
+    o.on('-o', '--output PATH', 'Write the output to PATH.') do |path|
+      options.output = path
+    end
+
+    o.on('--module NAME', 'The name of the generated FFI module.') do |m|
+      options.module = m
+    end
+
+    o.on('-l', '--library NAME', 'The shared library name to load.') do |l|
+      options.library = l
+    end
+
+    o.on('-t', '--typedef TYPE=NAME') do |v|
+      type, name = v.split('=')
+      options.typedefs[name] = type if type && name
+    end
+  end
+
+  options.sources = opts.parse!(ARGV)
+
+  unless options.mode == :help
+    raise ArgumentError.new("No sources found.") unless options.sources.size > 0
+  end
+
+  case options.mode
+    when :ffi then write_ffi(options.module, options.library, options.output, options.sources, options.typedefs)
+    when :header then write_header(options.output, options.sources)
+    when :source then write_source(options.output, options.sources[0])
+    else puts(opts.help)
+  end
+end