iruby 0.2.8 → 0.6.0

data/ci/Dockerfile.base.erb

@@ -0,0 +1,41 @@
+FROM rubylang/ruby:<%= ruby_version %>-bionic
+
+ADD ci/requirements.txt /tmp
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+               libczmq-dev \
+               python3 \
+               python3-pip \
+               python3-setuptools \
+               libpython3.6 \
+    && pip3 install wheel \
+    && pip3 install -r /tmp/requirements.txt \
+    && rm -f /tmp/requirements.txt
+
+# ZeroMQ version 4.1.6 and CZMQ version 3.0.2 for rbczmq
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+               build-essential \
+               file \
+               wget \
+    && cd /tmp \
+    && wget https://github.com/zeromq/zeromq4-1/releases/download/v4.1.6/zeromq-4.1.6.tar.gz \
+    && wget https://archive.org/download/zeromq_czmq_3.0.2/czmq-3.0.2.tar.gz \
+    && tar xf zeromq-4.1.6.tar.gz \
+    && tar xf czmq-3.0.2.tar.gz \
+    && \
+    ( \
+        cd zeromq-4.1.6 \
+        && ./configure \
+        && make install \
+    ) \
+    && \
+    ( \
+        cd czmq-3.0.2 \
+        && wget -O 1.patch https://github.com/zeromq/czmq/commit/2594d406d8ec6f54e54d7570d7febba10a6906b2.diff \
+        && wget -O 2.patch https://github.com/zeromq/czmq/commit/b651cb479235751b22b8f9a822a2fc6bc1be01ab.diff \
+        && cat *.patch | patch -p1 \
+        && ./configure \
+        && make install \
+    )

data/ci/Dockerfile.main.erb

@@ -0,0 +1,7 @@
+FROM iruby-test-base:ruby-<%= ruby_version %>
+
+RUN gem install cztop
+RUN mkdir -p /iruby
+ADD . /iruby
+WORKDIR /iruby
+RUN bundle install

data/ci/requirements.txt

@@ -0,0 +1 @@
+jupyter-console>=6.0.0

data/docker/setup.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -ex
+
+apt-get update
+apt-get install -y --no-install-recommends \
+        libczmq-dev \
+        python3 \
+        python3-pip \
+        python3-setuptools \
+        python3-wheel
+
+cd /tmp/iruby
+bundle install --with test --without plot
+pip3 install jupyter

data/docker/test.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+set -ex
+
+cd /tmp/iruby
+bundle install --with test --without plot
+bundle exec rake test

data/iruby.gemspec

@@ -1,33 +1,29 @@
-# coding: utf-8
 require_relative 'lib/iruby/version'
-require 'date'
 
 Gem::Specification.new do |s|
   s.name          = 'iruby'
-  s.date          = Date.today.to_s
   s.version       = IRuby::VERSION
   s.authors       = ['Daniel Mendler', 'The SciRuby developers']
   s.email         = ['mail@daniel-mendler.de']
-  s.summary       = 'Ruby Kernel for Jupyter/IPython'
-  s.description   = 'A Ruby kernel for Jupyter/IPython frontends (e.g. notebook). Try it at try.jupyter.org.'
+  s.summary       = 'Ruby Kernel for Jupyter'
+  s.description   = 'A Ruby kernel for Jupyter environment. Try it at try.jupyter.org.'
   s.homepage      = 'https://github.com/SciRuby/iruby'
   s.license       = 'MIT'
 
-  s.files         = `git ls-files`.split($/)
+  s.files         = `git ls-files`.split($INPUT_RECORD_SEPARATOR)
   s.executables   = s.files.grep(%r{^bin/}) { |f| File.basename(f) }
   s.test_files    = s.files.grep(%r{^test/})
-  s.require_paths = %w(lib)
+  s.require_paths = %w[lib]
 
-  m = "Consider installing the optional dependencies to get additional functionality:\n"
-  File.read('Gemfile').scan(/gem\s+'(.*?)'/) { m << "  * #{$1}\n" }
-  s.post_install_message = m << "\n"
+  s.required_ruby_version = '>= 2.3.0'
 
-  s.required_ruby_version = '>= 2.1.0'
+  s.add_dependency 'data_uri', '~> 0.1'
+  s.add_dependency 'ffi-rzmq'
+  s.add_dependency 'mime-types', '>= 3.3.1'
+  s.add_dependency 'multi_json', '~> 1.11'
 
-  s.add_development_dependency 'rake', '~> 10.4'
-  s.add_development_dependency 'minitest', '~> 5.6'
-
-  s.add_runtime_dependency 'bond', '~> 0.5'
-  s.add_runtime_dependency 'multi_json', '~> 1.11'
-  s.add_runtime_dependency 'mimemagic', '~> 0.3'
+  s.add_development_dependency 'pycall', '>= 1.2.1'
+  s.add_development_dependency 'rake'
+  s.add_development_dependency 'test-unit'
+  s.add_development_dependency 'test-unit-rr'
 end

data/lib/iruby.rb

@@ -1,25 +1,33 @@
-require 'mimemagic'
+require 'mime/types'
 require 'multi_json'
 require 'securerandom'
 require 'openssl'
 require 'tempfile'
 require 'set'
+
 require 'iruby/version'
+require 'iruby/jupyter'
+require 'iruby/event_manager'
 require 'iruby/kernel'
 require 'iruby/backend'
 require 'iruby/ostream'
+require 'iruby/input'
 require 'iruby/formatter'
 require 'iruby/utils'
 require 'iruby/display'
 require 'iruby/comm'
-require 'iruby/session/mixin'
 
-begin
-  require 'iruby/session/rbczmq'
-rescue LoadError
+if ENV.fetch('IRUBY_OLD_SESSION', false)
+  require 'iruby/session/mixin'
   begin
     require 'iruby/session/ffi_rzmq'
   rescue LoadError
-    STDERR.puts "You should install rbczmq or ffi_rzmq before running iruby notebook. See README."
+    begin
+      require 'iruby/session/cztop'
+    rescue LoadError
+      STDERR.puts "Please install ffi-rzmq or cztop before running iruby. See README."
+    end
   end
+else
+  require 'iruby/session'
 end

data/lib/iruby/backend.rb

@@ -1,10 +1,9 @@
 module IRuby
   In, Out = [nil], [nil]
-  ::In, ::Out = In, Out
 
   module History
     def eval(code, store_history)
-      b = TOPLEVEL_BINDING
+      b = eval_binding
 
       b.local_variable_set(:_ih, In)  unless b.local_variable_defined?(:_ih)
       b.local_variable_set(:_oh, Out) unless b.local_variable_defined?(:_oh)
@@ -33,39 +32,74 @@ module IRuby
   end
 
   class PlainBackend
+    attr_reader :eval_path
     prepend History
 
     def initialize
-      require 'bond'
-      Bond.start(debug: true)
+      require 'irb'
+      require 'irb/completion'
+      IRB.setup(nil)
+      @main = TOPLEVEL_BINDING.eval("self").dup
+      @workspace = IRB::WorkSpace.new(@main)
+      @irb = IRB::Irb.new(@workspace)
+      @eval_path = @irb.context.irb_path
+      IRB.conf[:MAIN_CONTEXT] = @irb.context
+    end
+
+    def eval_binding
+      @workspace.binding
     end
 
     def eval(code, store_history)
-      TOPLEVEL_BINDING.eval(code)
+      @irb.context.evaluate(code, 0)
+      @irb.context.last_value
     end
 
     def complete(code)
-      Bond.agent.call(code, code)
+      IRB::InputCompletor::CompletionProc.call(code)
     end
   end
 
   class PryBackend
+    attr_reader :eval_path
     prepend History
 
     def initialize
       require 'pry'
+      Pry.memory_size = 3
       Pry.pager = false # Don't use the pager
       Pry.print = proc {|output, value|} # No result printing
       Pry.exception_handler = proc {|output, exception, _| }
+      @eval_path = Pry.eval_path
       reset
     end
 
+    def eval_binding
+      TOPLEVEL_BINDING
+    end
+
     def eval(code, store_history)
+      Pry.current_line = 1
       @pry.last_result = nil
       unless @pry.eval(code)
         reset
         raise SystemExit
       end
+
+      # Pry::Code.complete_expression? return false
+      if !@pry.eval_string.empty?
+        syntax_error = @pry.eval_string
+        @pry.reset_eval_string
+        @pry.evaluate_ruby(syntax_error)
+
+      # Pry::Code.complete_expression? raise SyntaxError
+      # evaluate again for current line number
+      elsif @pry.last_result_is_exception? &&
+              @pry.last_exception.is_a?(SyntaxError) &&
+              @pry.last_exception.is_a?(Pry::UserError)
+         @pry.evaluate_ruby(code)
+      end
+
       raise @pry.last_exception if @pry.last_result_is_exception?
       @pry.push_initial_binding unless @pry.current_binding # ensure that we have a binding
       @pry.last_result
@@ -76,7 +110,7 @@ module IRuby
     end
 
     def reset
-      @pry = Pry.new(output: $stdout, target: TOPLEVEL_BINDING)
+      @pry = Pry.new(output: $stdout, target: eval_binding)
     end
   end
 end

data/lib/iruby/command.rb

@@ -1,3 +1,5 @@
+require 'iruby/jupyter'
+
 require 'fileutils'
 require 'multi_json'
 
@@ -6,15 +8,18 @@ module IRuby
     def initialize(args)
       @args = args
 
-      ipython_dir = ENV['IPYTHONDIR'] || '~/.ipython'
-      @args.each do |arg|
-        ipython_dir = $1 if arg =~ /\A--ipython-dir=(.*)\Z/
-      end
-      @kernel_dir = File.join(File.expand_path(ipython_dir), 'kernels', 'ruby')
-      @kernel_file = File.join(@kernel_dir, 'kernel.json')
+      @ipython_dir = File.expand_path("~/.ipython")
+      @kernel_dir = resolve_kernelspec_dir.freeze
+      @kernel_file = File.join(@kernel_dir, 'kernel.json').freeze
       @iruby_path = File.expand_path $0
     end
 
+    attr_reader :ipython_dir, :kernel_dir, :kernel_file
+
+    def ipython_kernel_dir
+      File.join(File.expand_path(@ipython_dir), 'kernels', 'ruby')
+    end
+
     def run
       case @args.first
       when 'version', '-v', '--version'
@@ -23,11 +28,12 @@ module IRuby
       when 'help', '-h', '--help'
         print_help
       when 'register'
-        if registered_iruby_path && !@args.include?('--force')
+        force_p = @args.include?('--force')
+        if registered_iruby_path && !force_p
           STDERR.puts "#{@kernel_file} already exists!\nUse --force to force a register."
           exit 1
         end
-        register_kernel
+        register_kernel(force_p)
       when 'unregister'
         unregister_kernel
       when 'kernel'
@@ -39,6 +45,38 @@ module IRuby
 
     private
 
+    def resolve_kernelspec_dir
+      if ENV.has_key?('JUPYTER_DATA_DIR')
+        if ENV.has_key?('IPYTHONDIR')
+          warn 'both JUPYTER_DATA_DIR and IPYTHONDIR are supplied; IPYTHONDIR is ignored.'
+        end
+        if @args.find {|x| /\A--ipython-dir=/ =~ x }
+          warn 'both JUPYTER_DATA_DIR and --ipython-dir are supplied; --ipython-dir is ignored.'
+        end
+        jupyter_data_dir = ENV['JUPYTER_DATA_DIR']
+        return File.join(jupyter_data_dir, 'kernels', 'ruby')
+      end
+
+      if ENV.has_key?('IPYTHONDIR')
+        warn 'IPYTHONDIR is deprecated. Use JUPYTER_DATA_DIR instead.'
+        ipython_dir = ENV['IPYTHONDIR']
+      end
+
+      @args.each do |arg|
+        next unless /\A--ipython-dir=(.*)\Z/ =~ arg
+        ipython_dir = Regexp.last_match[1]
+        warn '--ipython-dir is deprecated. Use JUPYTER_DATA_DIR environment variable instead.'
+        break
+      end
+
+      if ipython_dir
+        @ipython_dir = ipython_dir
+        ipython_kernel_dir
+      else
+        File.join(Jupyter.kernelspec_dir, 'ruby')
+      end
+    end
+
     def print_help
       puts %{
 Usage:
@@ -56,6 +94,8 @@ Try `ipython help` for more information.
     def run_kernel
       require 'iruby/logger'
       IRuby.logger = MultiLogger.new(*Logger.new(STDOUT))
+      STDOUT.sync = true # FIXME: This can make the integration test.
+
       @args.reject! {|arg| arg =~ /\A--log=(.*)\Z/ && IRuby.logger.loggers << Logger.new($1) }
       IRuby.logger.level = @args.delete('--debug') ? Logger::DEBUG : Logger::INFO
 
@@ -64,7 +104,7 @@ Try `ipython help` for more information.
       Dir.chdir(working_dir) if working_dir
 
       require boot_file if boot_file
-      check_bundler {|e| IRuby.logger.warn "Could not load bundler: #{e.message}\n#{e.backtrace.join("\n")}" }
+      check_bundler {|e| IRuby.logger.warn "Could not load bundler: #{e.message}" }
 
       require 'iruby'
       Kernel.new(config_file).run
@@ -95,7 +135,7 @@ Try `ipython help` for more information.
     end
 
     def check_registered_kernel
-      if kernel = registered_iruby_path
+      if (kernel = registered_iruby_path)
         STDERR.puts "#{@iruby_path} differs from registered path #{registered_iruby_path}.
 This might not work. Run 'iruby register --force' to fix it." if @iruby_path != kernel
       else
@@ -106,14 +146,19 @@ This might not work. Run 'iruby register --force' to fix it." if @iruby_path !=
     def check_bundler
       require 'bundler'
       raise %q{iruby is missing from Gemfile. This might not work.
-Add `gem 'iruby'` to your Gemfile to fix it.} unless Bundler.definition.dependencies.any? {|s| s.name == 'iruby' }
+Add `gem 'iruby'` to your Gemfile to fix it.} unless Bundler.definition.specs.any? {|s| s.name == 'iruby' }
       Bundler.setup
     rescue LoadError
     rescue Exception => e
       yield(e)
     end
 
-    def register_kernel
+    def register_kernel(force_p=false)
+      if force_p
+        unregister_kernel_in_ipython_dir
+      else
+        return unless check_existing_kernel_in_ipython_dir
+      end
       FileUtils.mkpath(@kernel_dir)
       unless RUBY_PLATFORM =~ /mswin(?!ce)|mingw|cygwin/
         File.write(@kernel_file, MultiJson.dump(argv: [ @iruby_path, 'kernel', '{connection_file}' ],
@@ -127,6 +172,13 @@ Add `gem 'iruby'` to your Gemfile to fix it.} unless Bundler.definition.dependen
       FileUtils.copy(Dir[File.join(__dir__, 'assets', '*')], @kernel_dir) rescue nil
     end
 
+    def check_existing_kernel_in_ipython_dir
+      return true unless File.file?(File.join(ipython_kernel_dir, 'kernel.json'))
+      warn "IRuby kernel file already exists in the deprecated IPython's data directory."
+      warn "Using --force, you can replace the old kernel file with the new one in Jupyter's data directory."
+      false
+    end
+
     def registered_iruby_path
       File.exist?(@kernel_file) && MultiJson.load(File.read(@kernel_file))['argv'].first
     end
@@ -134,5 +186,9 @@ Add `gem 'iruby'` to your Gemfile to fix it.} unless Bundler.definition.dependen
     def unregister_kernel
       FileUtils.rm_rf(@kernel_dir)
     end
+
+    def unregister_kernel_in_ipython_dir
+      FileUtils.rm_rf(ipython_kernel_dir)
+    end
   end
 end

data/lib/iruby/display.rb

@@ -11,7 +11,10 @@ module IRuby
         obj = obj.object
 
         fuzzy_mime = options[:format] # Treated like a fuzzy mime type
-        raise 'Invalid argument :format' unless !fuzzy_mime || String === fuzzy_mime
+        unless !fuzzy_mime || String === fuzzy_mime
+          raise 'Invalid argument :format'
+        end
+
         if exact_mime = options[:mime]
           raise 'Invalid argument :mime' unless String === exact_mime
           raise 'Invalid mime type' unless exact_mime.include?('/')
@@ -27,28 +30,49 @@ module IRuby
 
         # As a last resort, interpret string representation of the object
         # as the given mime type.
-        data[exact_mime] = protect(exact_mime, obj) if exact_mime && !data.any? {|m,_| exact_mime == m }
+        if exact_mime && data.none? { |m, _| exact_mime == m }
+          data[exact_mime] = protect(exact_mime, obj)
+        end
 
         data
       end
 
+      def clear_output(wait = false)
+        IRuby::Kernel.instance.session.send(:publish, :clear_output, wait: wait)
+      end
+
       private
 
       def protect(mime, data)
-        MimeMagic.new(mime).text? ? data.to_s : [data.to_s].pack('m0')
+        ascii?(mime) ? data.to_s : [data.to_s].pack('m0')
+      end
+
+      def ascii?(mime)
+        case mime
+        when "application/javascript"
+          # Special case for application/javascript.
+          # This needs because mime-types tells us application/javascript a non-text type.
+          true
+        else
+          MIME::Type.new(mime).ascii?
+        end
       end
 
       def render(data, obj, exact_mime, fuzzy_mime)
         # Filter matching renderer by object type
-        renderer = Registry.renderer.select {|r| r.match?(obj) }
+        renderer = Registry.renderer.select { |r| r.match?(obj) }
 
         matching_renderer = nil
 
         # Find exactly matching display by exact_mime
-        matching_renderer = renderer.find {|r| exact_mime == r.mime } if exact_mime
+        if exact_mime
+          matching_renderer = renderer.find { |r| exact_mime == r.mime }
+        end
 
         # Find fuzzy matching display by fuzzy_mime
-        matching_renderer ||= renderer.find {|r| r.mime && r.mime.include?(fuzzy_mime) } if fuzzy_mime
+        if fuzzy_mime
+          matching_renderer ||= renderer.find { |r| r.mime&.include?(fuzzy_mime) }
+        end
 
         renderer.unshift matching_renderer if matching_renderer
 
@@ -69,7 +93,8 @@ module IRuby
       attr_reader :object, :options
 
       def initialize(object, options)
-        @object, @options = object, options
+        @object = object
+        @options = options
       end
 
       class << self
@@ -87,10 +112,13 @@ module IRuby
     end
 
     class Renderer
-      attr_reader :match, :mime, :render, :priority
+      attr_reader :match, :mime, :priority
 
       def initialize(match, mime, render, priority)
-        @match, @mime, @render, @priority = match, mime, render, priority
+        @match = match
+        @mime = mime
+        @render = render
+        @priority = priority
       end
 
       def match?(obj)
@@ -110,7 +138,7 @@ module IRuby
         @renderer ||= []
       end
 
-      SUPPORTED_MIMES = %w(
+      SUPPORTED_MIMES = %w[
         text/plain
         text/html
         text/latex
@@ -118,7 +146,8 @@ module IRuby
         application/javascript
         image/png
         image/jpeg
-        image/svg+xml)
+        image/svg+xml
+      ]
 
       def match(&block)
         @match = block
@@ -127,7 +156,7 @@ module IRuby
       end
 
       def respond_to(name)
-        match {|obj| obj.respond_to?(name) }
+        match { |obj| obj.respond_to?(name) }
       end
 
       def type(&block)
@@ -149,7 +178,7 @@ module IRuby
 
       def format(mime = nil, &block)
         renderer << Renderer.new(@match, mime, block, @priority)
-        renderer.sort_by! {|r| -r.priority }
+        renderer.sort_by! { |r| -r.priority }
 
         # Decrease priority implicitly for all formats
         # which are added later for a type.
@@ -165,7 +194,19 @@ module IRuby
           LaTeX.vector(obj.to_a)
       end
 
+      type { Numo::NArray }
+      format 'text/plain', &:inspect
+      format 'text/latex' do |obj|
+        obj.ndim == 2 ?
+        LaTeX.matrix(obj, obj.shape[0], obj.shape[1]) :
+          LaTeX.vector(obj.to_a)
+      end
+      format 'text/html' do |obj|
+        HTML.table(obj.to_a)
+      end
+
       type { NArray }
+      format 'text/plain', &:inspect
       format 'text/latex' do |obj|
         obj.dim == 2 ?
         LaTeX.matrix(obj.transpose(1, 0), obj.shape[1], obj.shape[0]) :
@@ -227,37 +268,41 @@ module IRuby
 
       match do |obj|
         defined?(Magick::Image) && Magick::Image === obj ||
-        defined?(MiniMagick::Image) && MiniMagick::Image === obj
+          defined?(MiniMagick::Image) && MiniMagick::Image === obj
       end
       format 'image' do |obj|
         format = obj.format || 'PNG'
-        [format == 'PNG' ? 'image/png' : 'image/jpeg', obj.to_blob {|i| i.format = format }]
+        [format == 'PNG' ? 'image/png' : 'image/jpeg', obj.to_blob { |i| i.format = format }]
       end
 
-      type { Gruff::Base }
-      format 'image/png' do |obj|
-        obj.to_blob
+      match do |obj|
+        defined?(Vips::Image) && Vips::Image === obj
+      end
+      format do |obj|
+        # handles Vips::Error, vips_image_get: field "vips-loader" not found
+        loader = obj.get('vips-loader') rescue nil
+        if loader == 'jpegload'
+          ['image/jpeg', obj.write_to_buffer('.jpg')]
+        else
+          # falls back to png for other/unknown types
+          ['image/png', obj.write_to_buffer('.png')]
+        end
       end
 
+      type { Gruff::Base }
+      format 'image/png', &:to_blob
+
       respond_to :to_html
-      format 'text/html' do |obj|
-        obj.to_html
-      end
+      format 'text/html', &:to_html
 
       respond_to :to_latex
-      format 'text/latex' do |obj|
-        obj.to_latex
-      end
+      format 'text/latex', &:to_latex
 
       respond_to :to_tex
-      format 'text/latex' do |obj|
-        obj.to_tex
-      end
+      format 'text/latex', &:to_tex
 
       respond_to :to_javascript
-      format 'text/javascript' do |obj|
-        obj.to_javascript
-      end
+      format 'text/javascript', &:to_javascript
 
       respond_to :to_svg
       format 'image/svg+xml' do |obj|
@@ -266,21 +311,17 @@ module IRuby
       end
 
       respond_to :to_iruby
-      format do |obj|
-        obj.to_iruby
-      end
+      format(&:to_iruby)
 
-      match {|obj| obj.respond_to?(:path) && File.readable?(obj.path) }
+      match { |obj| obj.respond_to?(:path) && obj.method(:path).arity == 0 && File.readable?(obj.path) }
       format do |obj|
-        mime = MimeMagic.by_path(obj.path).to_s
+        mime = MIME::Types.of(obj.path).first.to_s
         [mime, File.read(obj.path)] if SUPPORTED_MIMES.include?(mime)
       end
 
       type { Object }
-      priority -1000
-      format 'text/plain' do |obj|
-        obj.inspect
-      end
+      priority(-1000)
+      format 'text/plain', &:inspect
     end
   end
 end