easy-serve 0.9 → 0.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2d1ef7c230f25e932ca33c4acbf8836789aeba20
-  data.tar.gz: 6003abf5645a32b1aadee49e711fbed4cfe8a8f4
+  metadata.gz: d74d8eda371150176555b9637f8e20b237edf068
+  data.tar.gz: ae3c58392fe3da0cb5e316f814a2a3b60e171087
 SHA512:
-  metadata.gz: 0618e996740b75fefef23d2fda92fc2f39a1e13d8e6d66d6d18c61f613c44aefd7419e49587b909bbc2c1aa86e9e605588c4f4f59d6b1311dfe18a87da77c448
-  data.tar.gz: 579718d353ad19562bb10a0dfdd0e9574476eeba4116844328a04e53ee0b13bf454cafa94a8194419a29eab912342e42297b2c778edf3c542cc623f0fbfcf2df
+  metadata.gz: 299b6abc59f540f9dd71f53d933ff5638cba9757a07fc9290c3bae5599868a3a9414e59c988d126582bd2f87a414fcff3dfcd7bfa3497f209bac8a894ae24919
+  data.tar.gz: fa63d61b24de5644289ac3711d15a25f1b869c547b46bbc733c5da17ca4849a2d280d7d44625f10648aadac6befda81fa35519c1d4a0bd5af6fd9489ce3c5a4d

data/README.md

@@ -2,3 +2,16 @@ easy-serve
 ==========
 
 Framework for starting tcp/unix services and connected clients under one parent process and on remote hosts.
+
+use cases
+---------
+
+1. start some procs with unix sockets established among them and
+   clean up afterwards [simple](examples/simple.rb) [multi](examples/multi.rb)
+
+2. ditto but with tcp and possibly [remote](examples/remote-eval.rb)
+
+3. ditto but through ssh [tunnels](examples/remote-eval.rb)
+
+4. ditto but where the tunnel is set up by the remote client, without
+   special assistance from the server [examples/tunnel](examples/tunnel)

data/examples/tunnel/client.rb

@@ -0,0 +1,33 @@
+require 'easy-serve/remote'
+
+services_file = ARGV.shift
+
+unless services_file
+  abort <<-END
+
+    Usage: #$0 services.yaml
+
+    Reads the yaml file and tunnels to the service. Note that
+    the filename may be remote, like host:path, so the lazy way
+    to run the example is:
+
+        host1$ ruby server.rb sv
+
+        host2$ ruby client.rb host1:path/to/sv
+
+  END
+end
+
+EasyServe.start services_file: services_file do |ez|
+  log = ez.log
+  log.level = Logger::INFO
+  log.formatter = nil if $VERBOSE
+  
+  ez.tunnel_to_remote_services
+
+  ez.child "hello-service" do |conn|
+    log.progname = "client 1"
+    log.info conn.read
+    conn.write "hello from #{log.progname}"
+  end
+end

data/examples/tunnel/server.rb

@@ -0,0 +1,43 @@
+require 'easy-serve'
+
+services_file = ARGV.shift
+
+unless services_file
+  abort <<-END
+
+    Usage: #$0 services.yaml
+
+    Creates the yaml file and sets up the service. The service is
+    listening only on localhost, so remote clients must use tunnels.
+    Does not set up any tunnels or remote clients. See client.rb.
+
+  END
+end
+
+EasyServe.start services_file: services_file do |ez|
+  log = ez.log
+  log.level = Logger::INFO
+  log.formatter = nil if $VERBOSE
+
+  ez.start_services do
+    host = "localhost" # no remote access, except by tunnel
+    ez.service "hello-service", :tcp, bind_host: host do |svr|
+      Thread.new do
+        loop do
+          Thread.new(svr.accept) do |conn|
+            log.info "accepted connection from #{conn.inspect}"
+            conn.write "hello from #{log.progname}"
+            log.info "wrote greeting"
+            conn.close_write
+            log.info "trying to read from #{conn.inspect}"
+            log.info "received: #{conn.read}"
+            conn.close
+          end
+        end
+      end
+    end
+  end
+
+  puts "PRESS RETURN TO STOP"
+  gets
+end

data/lib/easy-serve.rb

@@ -6,7 +6,7 @@ require 'fileutils'
 require 'easy-serve/service'
 
 class EasyServe
-  VERSION = "0.9"
+  VERSION = "0.10"
   
   class EasyFormatter < Logger::Formatter
     Format = "%s: %s: %s\n"
@@ -35,6 +35,11 @@ class EasyServe
   attr_reader :services_file
   attr_reader :interactive
   
+  # Is this a sibling process, started by the same parent process that
+  # started the services, even if started remotely?
+  # Implies not owner, but not conversely.
+  attr_reader :sibling
+
   def self.start(log: default_logger, **opts)
     ez = new(**opts, log: log)
     yield ez
@@ -45,6 +50,24 @@ class EasyServe
     ez.cleanup if ez
   end
 
+  # Options:
+  #
+  #   services_file: filename
+  #
+  #       name of file that server addresses are written to (if this process
+  #       is creating them) or read from (if this process is accessing them).
+  #       If not specified, services will be available to child processes,
+  #       but harder to access from other processes.
+  #
+  #       If the filename has a ':' in it, we assume that it is a remote
+  #       file, specified as [user@]host:path/to/file as in scp and rsync,
+  #       and attempt to read its contents over an ssh connection.
+  #
+  #   interactive: true|false
+  #
+  #       true means do not propagate ^C to child processes.
+  #       This is useful primarily when running in irb.
+  #
   def initialize **opts
     @services_file = opts[:services_file]
     @interactive = opts[:interactive]
@@ -52,6 +75,8 @@ class EasyServe
     @children = [] # pid
     @passive_children = [] # pid
     @owner = false
+    @sibling = true
+    @ssh_sessions = []
     @tmpdir = nil
     @services = opts[:services] # name => service
     
@@ -70,10 +95,21 @@ class EasyServe
   end
 
   def load_service_table
-    File.open(services_file) do |f|
-      YAML.load(f)
+    case services_file
+    when /\A(\S*):(.*)/
+      IO.popen ["ssh", $1, "cat #$2"], "r" do |f|
+        load_service_table_from_io f
+      end
+    else
+      File.open(services_file) do |f|
+        load_service_table_from_io f
+      end
     end
   end
+
+  def load_service_table_from_io io
+    YAML.load(io).tap {@sibling = false}
+  end
   
   def init_service_table
     @services ||= begin
@@ -161,6 +197,10 @@ class EasyServe
   end
   
   def host_name
+    EasyServe.host_name
+  end
+
+  def EasyServe.host_name
     @host_name ||= begin
       hn = Socket.gethostname
       begin
@@ -175,6 +215,10 @@ class EasyServe
       end
     end
   end
+
+  def EasyServe.ssh_supports_dynamic_ports_forwards
+    @ssh_6 ||= (Integer(`ssh -V 2>&1`[/OpenSSH_(\d)/i, 1]) >= 6 rescue false)
+  end
   
   MAX_TRIES = 10
 
@@ -246,4 +290,42 @@ class EasyServe
   def no_interrupt_if_interactive
     trap("INT") {} if interactive
   end
+
+  # Returns list of services that are accessible from +host+, setting
+  # up an ssh tunnel if specified. This is for the 'ssh -R' type of tunneling:
+  # a process, started remotely by some main process, needs to connect back to
+  # its siblings, other children of that main process. OpenSSH 6.0 or later is
+  # advised, but not necessary, for the tunnel option.
+  def accessible_services host, tunnel: false
+    tcp_svs = services.values.grep(TCPService)
+    return tcp_svs unless tunnel and host != "localhost" and host != "127.0.0.1"
+
+    require 'easy-serve/service/accessible'
+
+    tcp_svs.map do |service|
+      service, ssh_session = service.accessible(host, log)
+      @ssh_sessions << ssh_session # let GC close them
+      service
+    end
+  end
+
+  # Set up tunnels as needed and modify the service list so that connections
+  # will go to local endpoints in those cases. Call this method in non-sibling
+  # invocations, such as when the server file has been copied to a remote
+  # host and used to start a new client. This is for the 'ssh -L' type of
+  # tunneling: a process needs to connect to a cluster of remote EasyServe
+  # processes that already exist and do not know about this process.
+  def tunnel_to_remote_services
+    return if sibling
+
+    require 'easy-serve/service/tunnelled'
+
+    tunnelled_services = {}
+    services.each do |service_name, service|
+      service, ssh_session = service.tunnelled
+      tunnelled_services[service_name] = service
+      @ssh_sessions << ssh_session if ssh_session # let GC close them
+    end
+    @services = tunnelled_services
+  end
 end

data/lib/easy-serve/remote.rb

@@ -7,15 +7,15 @@ class EasyServe
     raise ArgumentError, "no host specified" unless host
 
     if opts[:eval]
-      require 'easy-serve/remote-eval'
+      require 'easy-serve/remote/eval'
       remote_eval(*service_names, host: host, **opts)
 
     elsif opts[:file]
-      require 'easy-serve/remote-run'
+      require 'easy-serve/remote/run'
       remote_run(*service_names, host: host, **opts)
 
     elsif block_given?
-      require 'easy-serve/remote-drb'
+      require 'easy-serve/remote/drb'
       remote_drb(*service_names, host: host, **opts, &Proc.new)
 
     else

data/lib/easy-serve/{remote-eval.rb → remote/eval.rb}

@@ -1,5 +1,4 @@
 require 'msgpack'
-require 'easy-serve/accessible-services'
 
 class EasyServe
   # useful simple cases in testing and in production, but long eval strings
@@ -20,7 +19,7 @@ class EasyServe
 
       IO.popen [
           "ssh", host, "ruby",
-          "-r", "easy-serve/remote-eval-mgr",
+          "-r", "easy-serve/remote/eval-mgr",
           "-e", "EasyServe.handle_remote_eval_messages"
         ],
         "w+" do |ssh|

data/lib/easy-serve/{remote-run.rb → remote/run.rb}

@@ -1,5 +1,4 @@
 require 'msgpack'
-require 'easy-serve/accessible-services'
 
 class EasyServe
   # useful in production, though it requires remote lib files to be set up.
@@ -18,7 +17,7 @@ class EasyServe
 
       IO.popen [
           "ssh", host, "ruby",
-          "-r", "easy-serve/remote-run-mgr",
+          "-r", "easy-serve/remote/run-mgr",
           "-e", "EasyServe.handle_remote_run_messages"
         ],
         "w+" do |ssh|

data/lib/easy-serve/service.rb

@@ -3,25 +3,6 @@ class EasyServe
   # Encapsulates current location and identity, including pid and address. A
   # Service object can be serialized to a remote process so it can #connect to
   # the service.
-  #
-  # The scheme for referencing hosts is as follows:
-  #
-  #      bind host  |              connect host
-  #                 +------------------------------------------------------
-  #                 |  local           remote TCP            SSH tunnel
-  #      -----------+------------------------------------------------------
-  #
-  #      localhost     'localhost'     X                     'localhost'
-  #
-  #      0.0.0.0       'localhost'     hostname(*)           'localhost'
-  #
-  #      hostname      hostname        hostname              'localhost'(**)
-  #
-  #      * use hostname as best guess, can override; append ".local" if
-  #        hostname not qualified
-  #
-  #      ** forwarding set up to hostname[.local] instead of localhost
-  #
   class Service
     attr_reader :name
     attr_reader :pid
@@ -111,14 +92,33 @@ class EasyServe
     end
   end
 
+  # The scheme for referencing TCP hosts is as follows:
+  #
+  #      bind host  |              connect host
+  #                 +------------------------------------------------------
+  #                 |  local           remote TCP            SSH tunnel
+  #      -----------+------------------------------------------------------
+  #
+  #      localhost     'localhost'     X                     'localhost'
+  #
+  #      0.0.0.0       'localhost'     hostname(*)           'localhost'
+  #
+  #      hostname      hostname        hostname              'localhost'(**)
+  #
+  #      * use hostname as best guess, can override; append ".local" if
+  #        hostname not qualified
+  #
+  #      ** forwarding set up to hostname[.local] instead of localhost
+  #
   class TCPService < Service
     SERVICE_CLASS[:tcp] = self
 
-    attr_reader :bind_host, :connect_host, :port
+    attr_reader :bind_host, :connect_host, :host, :port
 
-    def initialize name, bind_host: nil, connect_host: nil, port: 0
+    def initialize name, bind_host: nil, connect_host: nil, host: nil, port: 0
       super name
       @bind_host, @connect_host, @port = bind_host, connect_host, port
+      @host ||= EasyServe.host_name
     end
 
     def serve max_tries: 1, log: log
@@ -126,6 +126,7 @@ class EasyServe
         found_addr = svr.addr(false).values_at(2,1)
         log.debug "#{inspect} is listening at #{found_addr.join(":")}"
         @port = found_addr[1]
+        @bind_host ||= found_addr[0]
       end
     end
 
@@ -134,7 +135,7 @@ class EasyServe
     end
 
     def try_serve
-      TCPServer.new(bind_host, port)
+      TCPServer.new(bind_host, port || 0) # new(nil, nil) ==> error
     end
 
     def bump!

data/lib/easy-serve/service/accessible.rb

@@ -0,0 +1,81 @@
+require 'easy-serve/service'
+
+class EasyServe::TCPService
+  # Returns [service, ssh_session]. The service is modified based on self
+  # with tunneling from remote_host and ssh_session is the associated ssh pipe.
+  def accessible remote_host, log
+    service_host =
+      case bind_host
+      when nil, "localhost", "127.0.0.1", "0.0.0.0", /\A<any>\z/i
+        "localhost"
+      else
+        bind_host
+      end
+
+    fwd = "0:#{service_host}:#{port}"
+    remote_port = nil
+    ssh = nil
+    tries = 10
+
+    1.times do
+      if EasyServe.ssh_supports_dynamic_ports_forwards
+        remote_port = Integer(`ssh -O forward -R #{fwd} #{remote_host}`)
+      else
+        log.warn "Unable to set up dynamic ssh port forwarding. " +
+          "Please check if ssh -v is at least 6.0. " +
+          "Falling back to new ssh session."
+
+        code = <<-CODE
+          require 'socket'
+          svr = TCPServer.new "localhost", 0 # no rescue; error here is fatal
+          puts svr.addr[1]
+          svr.close
+        CODE
+
+        remote_port =
+          IO.popen ["ssh", remote_host, "ruby"], "w+" do |ruby|
+            ruby.puts code
+            ruby.close_write
+            Integer(ruby.gets)
+          end
+
+        cmd = [
+          "ssh", remote_host,
+          "-R", "#{remote_port}:#{service_host}:#{port}",
+          "echo ok && cat"
+        ]
+        ssh = IO.popen cmd, "w+"
+        ## how to tell if port in use and retry? ssh doesn't seem to fail,
+        ## or maybe it fails by printing a message on the remote side
+
+        ssh.sync = true
+        line = ssh.gets
+        unless line and line.chomp == "ok" # wait for forwarding
+          raise "Could not start ssh forwarding: #{cmd.join(" ")}"
+        end
+      end
+
+      if remote_port == 0
+        log.warn "race condition in ssh selection of remote_port"
+        tries -= 1
+        if tries > 0
+          sleep 0.1
+          log.info "retrying ssh selection of remote_port"
+          redo
+        end
+        raise "ssh did not assign remote_port"
+      end
+    end
+
+    # This breaks with multiple forward requests, and it would be too hard
+    # to coordinate among all requesting processes, so let's leave the
+    # forwarding open:
+    #at_exit {system "ssh -O cancel -R #{fwd} #{remote_host}"}
+
+    service =
+      self.class.new name, host: host,
+        bind_host: bind_host, connect_host: "localhost", port: remote_port
+
+    return [service, ssh]
+  end
+end

data/lib/easy-serve/service/tunnelled.rb

@@ -0,0 +1,53 @@
+require 'easy-serve/service'
+
+class EasyServe
+  class Service
+    def tunnelled(*)
+      [self, nil]
+    end
+  end
+
+  class TCPService
+    # Returns [service, ssh_session|nil]. The service is self and ssh_session
+    # is nil, unless tunneling is appropriate, in which case the returned
+    # service is the tunnelled one, and the ssh_session is the associated ssh
+    # pipe. This is for the 'ssh -L' type of tunneling: a process needs to
+    # connect to a cluster of remote EasyServe processes.
+    def tunnelled
+      return [self, nil] if
+        ["localhost", "127.0.0.1", EasyServe.host_name].include? host
+
+      if ["localhost", "127.0.0.1", "0.0.0.0"].include? bind_host
+        rhost = "localhost"
+      else
+        rhost = bind_host
+      end
+
+      svr = TCPServer.new "localhost", 0 # no rescue; error here is fatal
+      lport = svr.addr[1]
+      svr.close
+      ## why doesn't `ssh -L 0:host:port` work?
+
+      # possible alternative: ssh -f -N -o ExitOnForwardFailure: yes
+      cmd = [
+        "ssh", host,
+        "-L", "#{lport}:#{rhost}:#{port}",
+        "echo ok && cat"
+      ]
+      ssh = IO.popen cmd, "w+"
+      ## how to tell if lport in use and retry? ssh doesn't seem to fail,
+      ## or maybe it fails by printing a message on the remote side
+
+      ssh.sync = true
+      line = ssh.gets
+      unless line and line.chomp == "ok" # wait for forwarding
+        raise "Could not start ssh forwarding: #{cmd.join(" ")}"
+      end
+
+      service = TCPService.new name,
+        bind_host: bind_host, connect_host: 'localhost', host: host, port: lport
+
+      return [service, ssh]
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: easy-serve
 version: !ruby/object:Gem::Version
-  version: '0.9'
+  version: '0.10'
 platform: ruby
 authors:
 - Joel VanderWerf
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-12-18 00:00:00.000000000 Z
+date: 2013-12-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -35,25 +35,28 @@ extra_rdoc_files:
 files:
 - README.md
 - COPYING
-- lib/easy-serve.rb
-- lib/easy-serve/remote-eval-mgr.rb
-- lib/easy-serve/remote-run.rb
-- lib/easy-serve/remote-drb.rb
 - lib/easy-serve/remote.rb
-- lib/easy-serve/accessible-services.rb
-- lib/easy-serve/remote-eval.rb
-- lib/easy-serve/remote-run-mgr.rb
+- lib/easy-serve/service/accessible.rb
+- lib/easy-serve/service/tunnelled.rb
 - lib/easy-serve/service.rb
-- examples/simple.rb
-- examples/remote-manual.rb
-- examples/multi.rb
-- examples/remote-eval-passive.rb
+- lib/easy-serve/remote/eval.rb
+- lib/easy-serve/remote/run-mgr.rb
+- lib/easy-serve/remote/run.rb
+- lib/easy-serve/remote/drb.rb
+- lib/easy-serve/remote/eval-mgr.rb
+- lib/easy-serve.rb
 - examples/remote-run.rb
-- examples/remote-drb.rb
 - examples/remote-multi-server.rb
+- examples/passive.rb
+- examples/remote-eval-passive.rb
+- examples/remote-drb.rb
 - examples/remote-run-script.rb
+- examples/multi.rb
+- examples/remote-manual.rb
+- examples/simple.rb
+- examples/tunnel/client.rb
+- examples/tunnel/server.rb
 - examples/remote-eval.rb
-- examples/passive.rb
 homepage: https://github.com/vjoel/easy-serve
 licenses:
 - BSD

data/lib/easy-serve/accessible-services.rb

@@ -1,51 +0,0 @@
-class EasyServe
-  # Returns list of services that are accessible from +host+, setting
-  # up an ssh tunnel if specified. Note that OpenSSH 6.0 or later is required
-  # for the tunnel option.
-  def accessible_services host, tunnel: false
-    tcp_svs = services.values.grep(TCPService)
-    return tcp_svs unless tunnel and host != "localhost" and host != "127.0.0.1"
-
-    tcp_svs.map do |service|
-      service_host =
-        case service.bind_host
-        when nil, "localhost", "127.0.0.1", "0.0.0.0", /\A<any>\z/i
-          "localhost"
-        else
-          service.bind_host
-        end
-
-      fwd = "0:#{service_host}:#{service.port}"
-      remote_port = nil
-      tries = 10
-      1.times do
-        out = `ssh -O forward -R #{fwd} #{host}`
-        begin
-          remote_port = Integer(out)
-        rescue
-          log.error "Unable to set up dynamic ssh port forwarding. " +
-            "Please check if ssh -v is at least 6.0."
-          raise
-        end
-
-        if remote_port == 0
-          log.warn "race condition in ssh selection of remote_port"
-          tries -= 1
-          if tries > 0
-            sleep 0.1
-            log.info "retrying ssh selection of remote_port"
-            redo
-          end
-          raise "ssh did not assign remote_port"
-        end
-      end
-
-      # This breaks with multiple forward requests, and it would be too hard
-      # to coordinate among all requesting processes, so let's leave the
-      # forwarding open:
-      #at_exit {system "ssh -O cancel -R #{fwd} #{host}"}
-
-      TCPService.new service.name, connect_host: "localhost", port: remote_port
-    end
-  end
-end