iruby 0.7.3 → 0.8.0

data/lib/iruby/application.rb

@@ -0,0 +1,380 @@
+require "fileutils"
+require "json"
+require "optparse"
+require "rbconfig"
+require "singleton"
+
+require_relative "error"
+require_relative "kernel_app"
+
+module IRuby
+  class Application
+    include Singleton
+
+    # Set the application instance up.
+    def setup(argv=nil)
+      @iruby_executable = File.expand_path($PROGRAM_NAME)
+      parse_command_line(argv)
+    end
+
+    # Parse the command line arguments
+    #
+    # @param argv [Array<String>, nil] The array of arguments.
+    private def parse_command_line(argv)
+      argv = ARGV.dup if argv.nil?
+      @argv = argv  # save the original
+
+      case argv[0]
+      when "help"
+        # turn `iruby help notebook` into `iruby notebook -h`
+        argv = [*argv[1..-1], "-h"]
+      when "version"
+        # turn `iruby version` into `iruby -v`
+        argv = ["-v", *argv[1..-1]]
+      else
+        argv = argv.dup  # prevent to break @argv
+      end
+
+      opts = OptionParser.new
+      opts.program_name = "IRuby"
+      opts.version = ::IRuby::VERSION
+      opts.banner = "Usage: #{$PROGRAM_NAME} [options] [subcommand] [options]"
+
+      opts.on_tail("-h", "--help") do
+        print_help(opts)
+        exit
+      end
+
+      opts.on_tail("-v", "--version") do
+        puts opts.ver
+        exit
+      end
+
+      opts.order!(argv)
+
+      if argv.length == 0 || argv[0].start_with?("-")
+        # If no subcommand is given, we use the console
+        argv = ["console", *argv]
+      end
+
+      begin
+        parse_sub_command(argv) if argv.length > 0
+      rescue InvalidSubcommandError => err
+        $stderr.puts err.message
+        print_help(opts, $stderr)
+        abort
+      end
+    end
+
+    SUB_COMMANDS = {
+      "register" => "Register IRuby kernel.",
+      "unregister" => "Unregister the existing IRuby kernel.",
+      "kernel" => "Launch IRuby kernel",
+      "console" => "Launch jupyter console with IRuby kernel"
+    }.freeze.each_value(&:freeze)
+
+    private_constant :SUB_COMMANDS
+
+    private def parse_sub_command(argv)
+      sub_cmd, *sub_argv = argv
+      case sub_cmd
+      when *SUB_COMMANDS.keys
+        @sub_cmd = sub_cmd.to_sym
+        @sub_argv = sub_argv
+      else
+        raise InvalidSubcommandError.new(sub_cmd, sub_argv)
+      end
+    end
+
+    private def print_help(opts, out=$stdout)
+      out.puts opts.help
+      out.puts
+      out.puts "Subcommands"
+      out.puts "==========="
+      SUB_COMMANDS.each do |name, description|
+        out.puts "#{name}"
+        out.puts "    #{description}"
+      end
+    end
+
+    def run
+      case @sub_cmd
+      when :register
+        register_kernel(@sub_argv)
+      when :unregister
+        unregister_kernel(@sub_argv)
+      when :console
+        exec_jupyter(@sub_cmd.to_s, @sub_argv)
+      when :kernel
+        @sub_app = KernelApplication.new(@sub_argv)
+        @sub_app.run
+      else
+        raise "[IRuby][BUG] Unknown subcommand: #{@sub_cmd}; this must be treated in parse_command_line."
+      end
+    end
+
+    ruby_version_info = RUBY_VERSION.split('.')
+    DEFAULT_KERNEL_NAME = "ruby#{ruby_version_info[0]}".freeze
+    DEFAULT_DISPLAY_NAME = "Ruby #{ruby_version_info[0]} (iruby kernel)"
+
+    RegisterParams = Struct.new(
+      :name,
+      :display_name,
+      :profile,
+      :env,
+      :user,
+      :prefix,
+      :sys_prefix,
+      :force,
+      :ipython_dir
+    ) do
+      def initialize(*args)
+        super
+        self.name ||= DEFAULT_KERNEL_NAME
+        self.force = false
+        self.user = true
+      end
+    end
+
+    def register_kernel(argv)
+      params = parse_register_command_line(argv)
+
+      if params.name != DEFAULT_KERNEL_NAME
+        # `--name` is specified and `--display-name` is not
+        # default `params.display_name` to `params.name`
+        params.display_name ||= params.name
+      end
+
+      check_and_warn_kernel_in_default_ipython_directory(params)
+
+      if installed_kernel_exist?(params.name, params.ipython_dir)
+        unless params.force
+          $stderr.puts "IRuby kernel named `#{params.name}` already exists!"
+          $stderr.puts "Use --force to force register the new kernel."
+          exit 1
+        end
+      end
+
+      Dir.mktmpdir("iruby_kernel") do |tmpdir|
+        path = File.join(tmpdir, DEFAULT_KERNEL_NAME)
+        FileUtils.mkdir_p(path)
+
+        # Stage assets
+        assets_dir = File.expand_path("../assets", __FILE__)
+        FileUtils.cp_r(Dir.glob(File.join(assets_dir, "*")), path)
+
+        kernel_dict = {
+          "argv" => make_iruby_cmd(),
+          "display_name" => params.display_name || DEFAULT_DISPLAY_NAME,
+          "language" => "ruby",
+          "metadata" => {"debugger": false}
+        }
+
+        # TODO: Support params.profile
+        # TODO: Support params.env
+
+        kernel_content = JSON.pretty_generate(kernel_dict)
+        File.write(File.join(path, "kernel.json"), kernel_content)
+
+        args = ["--name=#{params.name}"]
+        args << "--user" if params.user
+        args << path
+
+        # TODO: Support params.prefix
+        # TODO: Support params.sys_prefix
+
+        system("jupyter", "kernelspec", "install", *args)
+      end
+    end
+
+    # Warn the existence of the IRuby kernel in the default IPython's kernels directory
+    private def check_and_warn_kernel_in_default_ipython_directory(params)
+      default_ipython_kernels_dir = File.expand_path("~/.ipython/kernels")
+      [params.name, "ruby"].each do |name|
+        if File.exist?(File.join(default_ipython_kernels_dir, name, "kernel.json"))
+          warn "IRuby kernel `#{name}` already exists in the deprecated IPython's data directory."
+        end
+      end
+    end
+
+    alias __system__ system
+
+    private def system(*cmdline, dry_run: false)
+      $stderr.puts "EXECUTE: #{cmdline.map {|x| x.include?(' ') ? x.inspect : x}.join(' ')}"
+      __system__(*cmdline) unless dry_run
+    end
+
+    private def installed_kernel_exist?(name, ipython_dir)
+      kernels_dir = resolve_kernelspec_dir(ipython_dir)
+      kernel_dir = File.join(kernels_dir, name)
+      File.file?(File.join(kernel_dir, "kernel.json"))
+    end
+
+    private def resolve_kernelspec_dir(ipython_dir)
+      if ENV.has_key?("JUPYTER_DATA_DIR")
+        if ENV.has_key?("IPYTHONDIR")
+          warn "both JUPYTER_DATA_DIR and IPYTHONDIR are supplied; IPYTHONDIR is ignored."
+        end
+        jupyter_data_dir = ENV["JUPYTER_DATA_DIR"]
+        return File.join(jupyter_data_dir, "kernels")
+      end
+
+      if ipython_dir.nil? && ENV.key?("IPYTHONDIR")
+        warn 'IPYTHONDIR is deprecated. Use JUPYTER_DATA_DIR instead.'
+        ipython_dir = ENV["IPYTHONDIR"]
+      end
+
+      if ipython_dir
+        File.join(ipython_dir, 'kerenels')
+      else
+        Jupyter.kernelspec_dir
+      end
+    end
+
+    private def make_iruby_cmd(executable: nil, extra_arguments: nil)
+      executable ||= default_executable
+      extra_arguments ||= []
+      [*Array(executable), "kernel", "-f", "{connection_file}", *extra_arguments]
+    end
+
+    private def default_executable
+      [RbConfig.ruby, @iruby_executable]
+    end
+
+    private def parse_register_command_line(argv)
+      opts = OptionParser.new
+      opts.banner = "Usage: #{$PROGRAM_NAME} register [options]"
+
+      params = RegisterParams.new
+
+      opts.on(
+        "--force",
+        "Force register a new kernel spec.  The existing kernel spec will be removed."
+      ) { params.force = true }
+
+      opts.on(
+        "--user",
+        "Register for the current user instead of system-wide."
+      ) { params.user = true }
+
+      opts.on(
+        "--name=VALUE", String,
+        "Specify a name for the kernelspec. This is needed to have multiple IRuby kernels at the same time."
+      ) {|v| params.name = v }
+
+      opts.on(
+        "--display-name=VALUE", String,
+        "Specify the display name for the kernelspec. This is helpful when you have multiple IRuby kernels."
+      ) {|v| kernel_display_name = v }
+
+      # TODO: --profile
+      # TODO: --prefix
+      # TODO: --sys-prefix
+      # TODO: --env
+
+      define_ipython_dir_option(opts, params)
+
+      opts.order!(argv)
+
+      params
+    end
+
+    UnregisterParams = Struct.new(
+      :names,
+      #:profile,
+      #:user,
+      #:prefix,
+      #:sys_prefix,
+      :ipython_dir,
+      :force,
+      :yes
+    ) do
+      def initialize(*args, **kw)
+        super
+        self.names = []
+        # self.user = true
+        self.force = false
+        self.yes = false
+      end
+    end
+
+    def unregister_kernel(argv)
+      params = parse_unregister_command_line(argv)
+      opts = []
+      opts << "-y" if params.yes
+      opts << "-f" if params.force
+      system("jupyter", "kernelspec", "uninstall", *opts, *params.names)
+    end
+
+    private def parse_unregister_command_line(argv)
+      opts = OptionParser.new
+      opts.banner = "Usage: #{$PROGRAM_NAME} unregister [options] NAME [NAME ...]"
+
+      params = UnregisterParams.new
+
+      opts.on(
+        "-f", "--force",
+        "Force removal, don't prompt for confirmation."
+      ) { params.force = true}
+
+      opts.on(
+        "-y", "--yes",
+        "Answer yes to any prompts."
+      ) { params.yes = true }
+
+      # TODO: --user
+      # TODO: --profile
+      # TODO: --prefix
+      # TODO: --sys-prefix
+
+      define_ipython_dir_option(opts, params)
+
+      opts.order!(argv)
+
+      params.names = argv.dup
+
+      params
+    end
+
+    def exec_jupyter(sub_cmd, argv)
+      opts = OptionParser.new
+      opts.banner = "Usage: #{$PROGRAM_NAME} unregister [options]"
+
+      kernel_name = resolve_installed_kernel_name(DEFAULT_KERNEL_NAME)
+      opts.on(
+        "--kernel=NAME", String,
+        "The name of the default kernel to start."
+      ) {|v| kernel_name = v }
+
+      opts.order!(argv)
+
+      opts = ["--kernel=#{kernel_name}"]
+      exec("jupyter", "console", *opts)
+    end
+
+    private def resolve_installed_kernel_name(default_name)
+      kernels = IO.popen(["jupyter", "kernelspec", "list", "--json"], "r", err: File::NULL) do |jupyter_out|
+        JSON.load(jupyter_out.read)
+      end
+      unless kernels["kernelspecs"].key?(default_name)
+        return "ruby" if kernels["kernelspecs"].key?("ruby")
+      end
+      default_name
+    end
+
+    private def define_ipython_dir_option(opts, params)
+      opts.on(
+        "--ipython-dir=DIR", String,
+        "Specify the IPython's data directory (DEPRECATED)."
+      ) do |v|
+        if ENV.key?("JUPYTER_DATA_DIR")
+          warn 'Both JUPYTER_DATA_DIR and --ipython-dir are supplied; --ipython-dir is ignored.'
+        else
+          warn '--ipython-dir is deprecated. Use JUPYTER_DATA_DIR environment variable instead.'
+        end
+
+        params.ipython_dir = v
+      end
+    end
+  end
+end

data/lib/iruby/backend.rb

@@ -1,6 +1,11 @@
 module IRuby
   In, Out = [nil], [nil]
 
+  class << self
+    attr_accessor :silent_assignment
+  end
+  self.silent_assignment = false
+
   module History
     def eval(code, store_history)
       b = eval_binding
@@ -52,8 +57,8 @@ module IRuby
     end
 
     def eval(code, store_history)
-      @irb.context.evaluate(code, 0)
-      @irb.context.last_value
+      @irb.context.evaluate(@irb.build_statement(code), 0)
+      @irb.context.last_value unless IRuby.silent_assignment && assignment_expression?(code)
     end
 
     def complete(code)
@@ -69,6 +74,10 @@ module IRuby
         wrapper_module.include(*args)
       end
     end
+
+    def assignment_expression?(code)
+      @irb.respond_to?(:assignment_expression?) && @irb.assignment_expression?(code)
+    end
   end
 
   class PryBackend

data/lib/iruby/error.rb

@@ -0,0 +1,12 @@
+module IRuby
+  class Error < StandardError
+  end
+
+  class InvalidSubcommandError < Error
+    def initialize(name, argv)
+      @name = name
+      @argv = argv
+      super("Invalid subcommand name: #{@name}")
+    end
+  end
+end

data/lib/iruby/jupyter.rb

@@ -2,8 +2,11 @@ module IRuby
   module Jupyter
     class << self
       # User's default kernelspec directory is described here:
-      #     https://jupyter.readthedocs.io/en/latest/projects/jupyter-directories.html
+      #     https://docs.jupyter.org/en/latest/use/jupyter-directories.html
       def default_data_dir
+        data_dir = ENV["JUPYTER_DATA_DIR"]
+        return data_dir if data_dir
+
         case
         when windows?
           appdata = windows_user_appdata

data/lib/iruby/kernel.rb

@@ -176,8 +176,9 @@ module IRuby
     # @private
     def execute_request(msg)
       code = msg[:content]['code']
-      store_history = msg[:content]['store_history']
       silent = msg[:content]['silent']
+      # https://jupyter-client.readthedocs.io/en/stable/messaging.html#execute
+      store_history = silent ? false : msg[:content].fetch('store_history', true)
 
       @execution_count += 1 if store_history
 
@@ -218,7 +219,8 @@ module IRuby
       end
 
       events.trigger(:post_execute)
-      events.trigger(:post_run_cell, result) unless silent
+      # **{} is for Ruby2.7. Gnuplot#to_hash returns an Array.
+      events.trigger(:post_run_cell, result, **{}) unless silent
 
       @session.send(:reply, :execute_reply, content)
     end

data/lib/iruby/kernel_app.rb

@@ -0,0 +1,108 @@
+module IRuby
+  class KernelApplication
+    def initialize(argv)
+      parse_command_line(argv)
+    end
+
+    def run
+      if @test_mode
+        dump_connection_file
+        return
+      end
+
+      run_kernel
+    end
+
+    DEFAULT_CONNECTION_FILE = "kernel-#{Process.pid}.json".freeze
+
+    private def parse_command_line(argv)
+      opts = OptionParser.new
+      opts.banner = "Usage: #{$PROGRAM_NAME} [options] [subcommand] [options]"
+
+      @connection_file = nil
+      opts.on(
+        "-f CONNECTION_FILE", String,
+        "JSON file in which to store connection info (default: kernel-<pid>.json)"
+      ) {|v| @connection_file = v }
+
+      @test_mode = false
+      opts.on(
+        "--test",
+        "Run as test mode; dump the connection file and exit."
+      ) { @test_mode = true }
+
+      @log_file = nil
+      opts.on(
+        "--log=FILE", String,
+        "Specify the log file."
+      ) {|v| @log_file = v }
+
+      @log_level = Logger::INFO
+      opts.on(
+        "--debug",
+        "Set log-level debug"
+      ) { @log_level = Logger::DEBUG }
+
+      opts.order!(argv)
+
+      if @connection_file.nil?
+        # Without -f option, the connection file is at the beginning of the rest arguments
+        if argv.length <= 3
+          @connection_file, @boot_file, @work_dir = argv
+        else
+          raise ArgumentError, "Too many commandline arguments"
+        end
+      else
+        if argv.length <= 2
+          @boot_file, @work_dir = argv
+        else
+          raise ArgumentError, "Too many commandline arguments"
+        end
+      end
+
+      @connection_file ||= DEFAULT_CONNECTION_FILE
+    end
+
+    private def dump_connection_file
+      puts File.read(@connection_file)
+    end
+
+    private def run_kernel
+      IRuby.logger = MultiLogger.new(*Logger.new(STDOUT))
+      STDOUT.sync = true # FIXME: This can make the integration test.
+
+      IRuby.logger.loggers << Logger.new(@log_file) unless @log_file.nil?
+      IRuby.logger.level = @log_level
+
+      if @work_dir
+        IRuby.logger.debug("iruby kernel") { "Change the working directory: #{@work_dir}" }
+        Dir.chdir(@work_dir)
+      end
+
+      if @boot_file
+        IRuby.logger.debug("iruby kernel") { "Load the boot file: #{@boot_file}" }
+        require @boot_file
+      end
+
+      check_bundler {|e| IRuby.logger.warn "Could not load bundler: #{e.message}" }
+
+      require "iruby"
+      Kernel.new(@connection_file).run
+    rescue Exception => e
+      IRuby.logger.fatal "Kernel died: #{e.message}\n#{e.backtrace.join("\n")}"
+      exit 1
+    end
+
+    private def check_bundler
+      require "bundler"
+      unless Bundler.definition.specs.any? {|s| s.name == "iruby" }
+        raise %{IRuby is missing from Gemfile. This might not work.  Add `gem "iruby"` in your Gemfile to fix it.}
+      end
+      Bundler.setup
+    rescue LoadError
+      # do nothing
+    rescue Exception => e
+      yield e
+    end
+  end
+end

data/lib/iruby/logger.rb

@@ -6,10 +6,20 @@ module IRuby
   end
 
   class MultiLogger < BasicObject
+    def initialize(*loggers, level: ::Logger::DEBUG)
+      @loggers = loggers
+      @level = level
+    end
+
     attr_reader :loggers
 
-    def initialize(*loggers)
-      @loggers = loggers
+    attr_reader :level
+
+    def level=(new_level)
+      @loggers.each do |l|
+        l.level = new_level
+      end
+      @level = new_level
     end
 
     def method_missing(name, *args, &b)

data/lib/iruby/session/mixin.rb

@@ -23,7 +23,7 @@ module IRuby
       idents, msg_list = frames[0..i-1], frames[i+1..-1]
 
       minlen = 5
-      raise 'malformed message, must have at least #{minlen} elements' unless msg_list.length >= minlen
+      raise "malformed message, must have at least #{minlen} elements" unless msg_list.length >= minlen
       s, header, parent_header, metadata, content, buffers = *msg_list
       raise 'Invalid signature' unless s == sign(msg_list[1..-1])
       {

data/lib/iruby/session.rb

@@ -2,6 +2,7 @@ require 'iruby/session_adapter'
 require 'iruby/session/mixin'
 
 require 'securerandom'
+require 'time'
 
 module IRuby
   class Session
@@ -79,6 +80,7 @@ module IRuby
       header = {
         msg_type: message_type,
         msg_id:   SecureRandom.uuid,
+        date:     Time.now.utc.iso8601,
         username: 'kernel',
         session:  @session_id,
         version:  '5.0'

data/lib/iruby/session_adapter.rb

@@ -40,14 +40,12 @@ module IRuby
 
     require_relative 'session_adapter/ffirzmq_adapter'
     require_relative 'session_adapter/cztop_adapter'
-    require_relative 'session_adapter/pyzmq_adapter'
     require_relative 'session_adapter/test_adapter'
 
     def self.select_adapter_class(name=nil)
       classes = {
         'ffi-rzmq' => SessionAdapter::FfirzmqAdapter,
         'cztop' => SessionAdapter::CztopAdapter,
-        # 'pyzmq' => SessionAdapter::PyzmqAdapter
         'test' => SessionAdapter::TestAdapter,
       }
       if (name ||= ENV.fetch('IRUBY_SESSION_ADAPTER', nil))
@@ -61,6 +59,9 @@ module IRuby
                   "Session adapter `#{name}` is unavailable"
           end
         end
+        if name == 'cztop'
+          warn "WARNING: cztop was deprecated and will be removed; Use ffi-rzmq, instead."
+        end
         return cls
       end
       classes.each_value do |cls|

data/lib/iruby/version.rb

@@ -1,3 +1,3 @@
 module IRuby
-  VERSION = '0.7.3'
+  VERSION = '0.8.0'
 end

data/test/helper.rb

@@ -2,18 +2,36 @@ require "iruby"
 require "json"
 require 'multi_json'
 require "pathname"
+require "rbconfig"
 require "test/unit"
 require "test/unit/rr"
 require "tmpdir"
 
-
 IRuby.logger = IRuby::MultiLogger.new(*Logger.new(STDERR, level: Logger::Severity::INFO))
 
 module IRubyTest
   class TestBase < Test::Unit::TestCase
+    TEST_DIR = File.expand_path("..", __FILE__).freeze
+    EXE_DIR = File.expand_path("../exe", TEST_DIR).freeze
+    LIB_DIR = File.expand_path("../lib", TEST_DIR).freeze
+
+    RUBY = RbConfig.ruby.freeze
+    IRUBY_PATH = File.join(EXE_DIR, "iruby").freeze
+
+    def iruby_command(*args)
+      [RUBY, "-I#{LIB_DIR}", IRUBY_PATH, *args]
+    end
+
     def self.startup
-      @__config_dir = Dir.mktmpdir("iruby-test")
+      @__work_dir = Dir.mktmpdir("iruby-test-data")
+
+      @__jupyter_data_dir = File.join(@__work_dir, "jupyter")
+      @__save_jupyter_data_dir = ENV["JUPYTER_DATA_DIR"]
+      ENV["JUPYTER_DATA_DIR"] = @__jupyter_data_dir
+
+      @__config_dir = File.join(@__work_dir, "config")
       @__config_path = Pathname.new(@__config_dir) + "config.json"
+      @__config_path.dirname.mkpath
       File.write(@__config_path, {
         control_port: 50160,
         shell_port: 57503,
@@ -30,7 +48,8 @@ module IRubyTest
     end
 
     def self.shutdown
-      FileUtils.remove_entry_secure(@__config_dir)
+      FileUtils.remove_entry_secure(@__work_dir)
+      ENV["JUPYTER_DATA_DIR"] = @__save_jupyter_data_dir
     end
 
     def self.test_config_filename