iruby 0.3 → 0.4.0

data/lib/iruby/jupyter.rb

@@ -0,0 +1,76 @@
+module IRuby
+  module Jupyter
+    class << self
+      # User's default kernelspec directory is described here:
+      #     https://jupyter.readthedocs.io/en/latest/projects/jupyter-directories.html
+      def default_data_dir
+        case
+        when windows?
+          appdata = windows_user_appdata
+          if !appdata.empty?
+            File.join(appdata, 'jupyter')
+          else
+            jupyter_config_dir = ENV.fetch('JUPYTER_CONFIG_DIR', File.expand_path('~/.jupyter'))
+            File.join(jupyter_config_dir, 'data')
+          end
+        when apple?
+          File.expand_path('~/Library/Jupyter')
+        else
+          xdg_data_home = ENV.fetch('XDG_DATA_HOME', '')
+          data_home = xdg_data_home[0] ? xdg_data_home : File.expand_path('~/.local/share')
+          File.join(data_home, 'jupyter')
+        end
+      end
+
+      def kernelspec_dir(data_dir=nil)
+        data_dir ||= default_data_dir
+        File.join(data_dir, 'kernels')
+      end
+
+      private
+
+      # returns %APPDATA%
+      def windows_user_appdata
+        check_windows
+        path = Fiddle::Pointer.malloc(2 * 300) # uint16_t[300]
+        csidl_appdata = 0x001a
+        case call_SHGetFolderPathW(Fiddle::NULL, csidl_appdata, Fiddle::NULL, 0, path)
+        when 0
+          len = (1 ... (path.size/2)).find {|i| path[2*i, 2] == "\0\0" }
+          path = path.to_str(2*len).encode(Encoding::UTF_8, Encoding::UTF_16LE)
+        else
+          ENV.fetch('APPDATA', '')
+        end
+      end
+
+      def call_SHGetFolderPathW(hwnd, csidl, hToken, dwFlags, pszPath)
+        require 'fiddle/import'
+        shell32 = Fiddle::Handle.new('shell32')
+        func = Fiddle::Function.new(
+          shell32['SHGetFolderPathW'],
+          [
+            Fiddle::TYPE_VOIDP,
+            Fiddle::TYPE_INT,
+            Fiddle::TYPE_VOIDP,
+            Fiddle::TYPE_INT,
+            Fiddle::TYPE_VOIDP
+          ],
+          Fiddle::TYPE_INT,
+          Fiddle::Importer.const_get(:CALL_TYPE_TO_ABI)[:stdcall])
+        func.(hwnd, csidl, hToken, dwFlags, pszPath)
+      end
+
+      def check_windows
+        raise 'the current platform is not Windows' unless windows?
+      end
+
+      def windows?
+        /mingw|mswin/ =~ RUBY_PLATFORM
+      end
+
+      def apple?
+        /darwin/ =~ RUBY_PLATFORM
+      end
+    end
+  end
+end

data/lib/iruby/kernel.rb

@@ -1,7 +1,6 @@
 module IRuby
   class Kernel
     RED = "\e[31m"
-    WHITE = "\e[37m"
     RESET = "\e[0m"
 
     class<< self
@@ -19,6 +18,8 @@ module IRuby
       $stdout = OStream.new(@session, :stdout)
       $stderr = OStream.new(@session, :stderr)
 
+      init_parent_process_poller
+
       @execution_count = 0
       @backend = create_backend
       @running = true
@@ -40,6 +41,7 @@ module IRuby
 
     def dispatch
       msg = @session.recv(:reply)
+      IRuby.logger.debug "Kernel#dispatch: msg = #{msg}"
       type = msg[:header]['msg_type']
       raise "Unknown message type: #{msg.inspect}" unless type =~ /comm_|_request\Z/ && respond_to?(type)
       begin
@@ -50,14 +52,14 @@ module IRuby
       end
     rescue Exception => e
       IRuby.logger.debug "Kernel error: #{e.message}\n#{e.backtrace.join("\n")}"
-      @session.send(:publish, :error, error_message(e))
+      @session.send(:publish, :error, error_content(e))
     end
 
     def kernel_info_request(msg)
       @session.send(:reply, :kernel_info_reply,
                     protocol_version: '5.0',
                     implementation: 'iruby',
-                    banner: "IRuby #{IRuby::VERSION}",
+                    banner: "IRuby #{IRuby::VERSION} (with #{@session.description})",
                     implementation_version: IRuby::VERSION,
                     language_info: {
                       name: 'ruby',
@@ -68,6 +70,7 @@ module IRuby
     end
 
     def send_status(status)
+      IRuby.logger.debug "Send status: #{status}"
       @session.send(:publish, :status, execution_state: status)
     end
 
@@ -88,8 +91,9 @@ module IRuby
       rescue SystemExit
         content[:payload] << { source: :ask_exit }
       rescue Exception => e
-        content = error_message(e)
+        content = error_content(e)
         @session.send(:publish, :error, content)
+        content[:status] = :error
       end
       @session.send(:reply, :execute_reply, content)
       @session.send(:publish, :execute_result,
@@ -98,12 +102,16 @@ module IRuby
                     execution_count: @execution_count) unless result.nil? || msg[:content]['silent']
     end
 
-    def error_message(e)
-      { status: :error,
-        ename: e.class.to_s,
+    def error_content(e)
+      { ename: e.class.to_s,
         evalue: e.message,
-        traceback: ["#{RED}#{e.class}#{RESET}: #{e.message}", *e.backtrace.map { |l| "#{WHITE}#{l}#{RESET}" }],
-        execution_count: @execution_count }
+        traceback: ["#{RED}#{e.class}#{RESET}: #{e.message}", *e.backtrace] }
+    end
+
+    def is_complete_request(msg)
+      # FIXME: the code completeness should be judged by using ripper or other Ruby parser
+      @session.send(:reply, :is_complete_reply,
+                    status: :unknown)
     end
 
     def complete_request(msg)
@@ -161,5 +169,37 @@ module IRuby
       Comm.comm[comm_id].handle_close(msg[:content]['data'])
       Comm.comm.delete(comm_id)
     end
+
+    private
+
+    def init_parent_process_poller
+      pid = ENV.fetch('JPY_PARENT_PID', 0).to_i
+      return unless pid > 1
+
+      case RUBY_PLATFORM
+      when /mswin/, /mingw/
+        # TODO
+      else
+        @parent_poller = start_parent_process_pollar_unix
+      end
+    end
+
+    def start_parent_process_pollar_unix
+      Thread.start do
+        IRuby.logger.warn("parent process poller thread started.")
+        loop do
+          begin
+            current_ppid = Process.ppid
+            if current_ppid == 1
+              IRuby.logger.warn("parent process appears to exited, shutting down.")
+              exit!(1)
+            end
+            sleep 1
+          rescue Errno::EINTR
+            # ignored
+          end
+        end
+      end
+    end
   end
 end

data/lib/iruby/ostream.rb

@@ -33,6 +33,10 @@ module IRuby
     alias_method :<<, :write
     alias_method :print, :write
 
+    def printf(*fmt)
+      write sprintf(*fmt)
+    end
+
     def puts(*lines)
       lines = [''] if lines.empty?
       lines.each { |s| write("#{s}\n")}

data/lib/iruby/session.rb

@@ -0,0 +1,116 @@
+require 'iruby/session_adapter'
+require 'iruby/session/mixin'
+
+require 'securerandom'
+
+module IRuby
+  class Session
+    include SessionSerialize
+
+    def initialize(config, adapter_name=nil)
+      @config = config
+      @adapter = create_session_adapter(config, adapter_name)
+
+      setup
+      setup_sockets
+      setup_heartbeat
+      setup_security
+    end
+
+    attr_reader :adapter, :config
+
+    def description
+      "#{@adapter.name} session adapter"
+    end
+
+    def setup
+    end
+
+    def setup_sockets
+      protocol, host = config.values_at('transport', 'ip')
+      shell_port = config['shell_port']
+      iopub_port = config['iopub_port']
+      stdin_port = config['stdin_port']
+
+      @shell_socket, @shell_port = @adapter.make_router_socket(protocol, host, shell_port)
+      @iopub_socket, @iopub_port = @adapter.make_pub_socket(protocol, host, iopub_port)
+      @stdin_socket, @stdin_port = @adapter.make_router_socket(protocol, host, stdin_port)
+
+      @sockets = {
+        publish: @iopub_socket,
+        reply:   @shell_socket,
+        stdin:   @stdin_socket
+      }
+    end
+
+    def setup_heartbeat
+      protocol, host = config.values_at('transport', 'ip')
+      hb_port = config['hb_port']
+      @hb_socket, @hb_port = @adapter.make_rep_socket(protocol, host, hb_port)
+      @heartbeat_thread = Thread.start do
+        begin
+          # NOTE: this loop is copied from CZTop's old session code
+          @adapter.heartbeat_loop(@hb_socket)
+        rescue Exception => e
+          IRuby.logger.fatal "Kernel heartbeat died: #{e.message}\n#{e.backtrace.join("\n")}"
+        end
+      end
+    end
+
+    def setup_security
+      @session_id = SecureRandom.uuid
+      unless config['key'].empty? || config['signature_scheme'].empty?
+        unless config['signature_scheme'] =~ /\Ahmac-/
+          raise "Unknown signature_scheme: #{config['signature_scheme']}"
+        end
+        digest_algorithm = config['signature_scheme'][/\Ahmac-(.*)\Z/, 1]
+        @hmac = OpenSSL::HMAC.new(config['key'], OpenSSL::Digest.new(digest_algorithm))
+      end
+    end
+
+    def send(socket_type, message_type, content)
+      sock = check_socket_type(socket_type)
+      idents = if socket_type == :reply && @last_recvd_msg
+                 @last_recvd_msg[:idents]
+               else
+                 message_type == :stream ? "stream.#{content[:name]}" : message_type
+               end
+      header = {
+        msg_type: message_type,
+        msg_id:   SecureRandom.uuid,
+        username: 'kernel',
+        session:  @session_id,
+        version:  '5.0'
+      }
+      @adapter.send(sock, serialize(idents, header, content))
+    end
+
+    def recv(socket_type)
+      sock = check_socket_type(socket_type)
+      data = @adapter.recv(sock)
+      @last_recvd_msg = unserialize(data)
+    end
+
+    def recv_input
+      sock = check_socket_type(:stdin)
+      data = @adapter.recv(sock)
+      unserialize(data)[:content]["value"]
+    end
+
+    private
+
+    def check_socket_type(socket_type)
+      case socket_type
+      when :publish, :reply, :stdin
+        @sockets[socket_type]
+      else
+        raise ArgumentError, "Invalid socket type #{socket_type}"
+      end
+    end
+
+    def create_session_adapter(config, adapter_name)
+      adapter_class = SessionAdapter.select_adapter_class(adapter_name)
+      adapter_class.new(config)
+    end
+  end
+end

data/lib/iruby/session/cztop.rb

@@ -36,6 +36,10 @@ module IRuby
       end
     end
 
+    def description
+      'old-stle session using cztop'
+    end
+
     # Build and send a message
     def send(socket, type, content)
       idents =

data/lib/iruby/session/rbczmq.rb

@@ -38,6 +38,10 @@ module IRuby
       end
     end
 
+    def description
+      'old-stle session using rbczmq'
+    end
+
     # Build and send a message
     def send(socket, type, content)
       idents =
@@ -53,7 +57,7 @@ module IRuby
         session:  @session,
         version:  '5.0'
       }
-      @sockets[socket].send_message(ZMQ::Message(*serialize(idents, header, content)))
+      @sockets[socket].send_message(ZMQ.Message(*serialize(idents, header, content)))
     end
 
     # Receive a message and decode it

data/lib/iruby/session_adapter.rb

@@ -0,0 +1,68 @@
+module IRuby
+  class SessionAdapterNotFound < RuntimeError; end
+
+  module SessionAdapter
+    class BaseAdapter
+      def self.available?
+        load_requirements
+        true
+      rescue LoadError
+        false
+      end
+
+      def initialize(config)
+        @config = config
+      end
+
+      def name
+        self.class.name[/::(\w+)Adapter\Z/, 1].downcase
+      end
+
+      def make_router_socket(protocol, host, port)
+        socket, port = make_socket(:ROUTER, protocol, host, port)
+        [socket, port]
+      end
+
+      def make_pub_socket(protocol, host, port)
+        socket, port = make_socket(:PUB, protocol, host, port)
+        [socket, port]
+      end
+
+      def make_rep_socket(protocol, host, port)
+        socket, port = make_socket(:REP, protocol, host, port)
+        [socket, port]
+      end
+    end
+
+    require_relative 'session_adapter/ffirzmq_adapter'
+    require_relative 'session_adapter/cztop_adapter'
+    require_relative 'session_adapter/rbczmq_adapter'
+    require_relative 'session_adapter/pyzmq_adapter'
+
+    def self.select_adapter_class(name=nil)
+      classes = {
+        'ffi-rzmq' => SessionAdapter::FfirzmqAdapter,
+        'cztop' => SessionAdapter::CztopAdapter,
+        'rbczmq' => SessionAdapter::RbczmqAdapter,
+        'pyzmq' => SessionAdapter::PyzmqAdapter
+      }
+      if (name ||= ENV.fetch('IRUBY_SESSION_ADAPTER', nil))
+        cls = classes[name]
+        unless cls.available?
+          if ENV['IRUBY_SESSION_ADAPTER']
+            raise SessionAdapterNotFound,
+                  "Session adapter `#{name}` from IRUBY_SESSION_ADAPTER is unavailable"
+          else
+            raise SessionAdapterNotFound,
+                  "Session adapter `#{name}` is unavailable"
+          end
+        end
+        return cls
+      end
+      classes.each_value do |cls|
+        return cls if cls.available?
+      end
+      raise SessionAdapterNotFound, "No session adapter is available"
+    end
+  end
+end

data/lib/iruby/session_adapter/cztop_adapter.rb

@@ -0,0 +1,45 @@
+module IRuby
+  module SessionAdapter
+    class CztopAdapter < BaseAdapter
+      def self.load_requirements
+        require 'cztop'
+      end
+
+      def send(sock, data)
+        sock << data
+      end
+
+      def recv(sock)
+        sock.receive
+      end
+
+      def heartbeat_loop(sock)
+        loop do
+          message = sock.receive
+          sock << message
+        end
+      end
+
+      private
+
+      def socket_type_class(type_symbol)
+        case type_symbol
+        when :ROUTER, :PUB, :REP
+          CZTop::Socket.const_get(type_symbol)
+        else
+          if CZTop::Socket.const_defined?(type_symbol)
+            raise ArgumentError, "Unsupported ZMQ socket type: #{type_symbol}"
+          else
+            raise ArgumentError, "Invalid ZMQ socket type: #{type_symbol}"
+          end
+        end
+      end
+
+      def make_socket(type_symbol, protocol, host, port)
+        uri = "#{protocol}://#{host}:#{port}"
+        socket_class = socket_type_class(type_symbol)
+        socket_class.new(uri)
+      end
+    end
+  end
+end