osc-vnc 1.0.6

data/lib/osc/vnc/listenable.rb

@@ -0,0 +1,45 @@
+require 'socket'
+
+
+module OSC
+  module VNC
+    # Mixin that adds the ability to create and read from a TCP server.
+    module Listenable
+      # Generate a TCP server that listens on a random port.
+      #
+      # @return [TCPServer] ruby TCPServer object listening on random port
+      def create_listen
+        listen_host = Socket.gethostname
+        listen_port = _get_port
+        begin
+          server = TCPServer.new(listen_host, listen_port)
+        rescue Errno::EADDRINUSE
+          listen_port = _get_port
+          retry
+        end
+        server
+      end
+
+      # Read the data received by the TCP server.
+      #
+      # @param [Hash] args the arguments to read data received by TCP server with
+      # @option args [TCPServer] :server the TCP server that is currently listening
+      # @return [String] the contents of the data received by the server
+      def read_listen(args)
+        server = args[:server]
+        client = server.accept  # wait for connection
+        client.read             # read complete response
+      end
+
+
+      private
+
+      # Get random number form 40,000 to 50,000.
+      #
+      # @return [Fixnum] a random number from 40,000 to 50,000
+      def _get_port
+        rand(40000..50000)
+      end
+    end
+  end
+end

data/lib/osc/vnc/scriptview.rb

@@ -0,0 +1,63 @@
+require 'fileutils'
+require 'mustache'
+require 'yaml'
+
+module OSC
+  module VNC
+    # Provides a view for the PBS batch script configured as a mustache
+    # template in templates/script/vnc.mustache. Extra options can be passed to
+    # this view and accessed directly in the mustache template.
+    class ScriptView < Mustache
+      self.template_path = SCRIPT_TEMPLATE_PATH
+
+      # @param type [Symbol] The script type defined in config/script.yml (:vnc or :server).
+      # @param cluster [String] The cluster the job will run on defined in config/script.yml.
+      # @param opts [Hash] The options used to construct PBS batch script view.
+      # @option opts [Symbol] :subtype The subtype (i.e., :default or :shared) outlined in config/script.yml.
+      def initialize(type, cluster, opts = {})
+        self.template_name = type
+
+        # Read in pre-configured options
+        script_cfg = YAML.load_file("#{CONFIG_PATH}/script.yml").fetch(type.to_s, {})
+        subtype_opts = script_cfg[opts[:subtype].to_s] || script_cfg['default']
+        cluster_opts = subtype_opts.fetch('cluster', {}).fetch(cluster, {})
+        context = subtype_opts.merge cluster_opts
+
+        @view_context = {}
+        context.each do |key, value|
+          @view_context[key.to_sym] = value
+        end
+        @view_context.merge! opts
+        @view_context[:cluster] = cluster
+      end
+
+      # Determine whether the script is valid or not
+      #
+      # @return [Boolean] Whether this is a valid script.
+      # @raise [InvalidPath] if { #xstartup } or { #outdir } do not correspond to actual file system locations
+      def valid?
+        raise InvalidPath, "xstartup script is not found" unless File.file?(xstartup)
+        raise InvalidPath, "output directory is a file" if File.file?(outdir)
+        FileUtils.mkdir_p(outdir)
+      end
+
+      # See if the method call exists as a key in @view_context.
+      #
+      # @param method_name the method name called
+      # @param arguments the arguments to the call
+      # @param block an optional block for the call
+      def method_missing(method_name, *arguments, &block)
+        @view_context.fetch(method_name) { super }
+      end
+
+      # Checks if the method responds to an instance method, or is able to
+      # proxy it to @view_context.
+      #
+      # @param method_name the method name to check
+      # @return [Boolean]
+      def respond_to_missing?(method_name, include_private = false)
+        @view_context.include?(method_name) || super
+      end
+    end
+  end
+end

data/lib/osc/vnc/session.rb

@@ -0,0 +1,116 @@
+require 'osc/vnc/listenable'
+
+module OSC
+  module VNC
+    # Provides a way for developers to create and submit VNC sessions to the
+    # OSC batch systems. Also, developers are now able to submit any server
+    # session to the batch systems whether it is VNC or not following the same
+    # principles as a VNC session.
+    class Session
+      include OSC::VNC::Listenable
+
+      # @return [PBS::Job] The job object used.
+      attr_reader :job
+
+      # @return [ScriptView] The batch script used.
+      attr_reader :script
+
+      # @param job [PBS::Job] The job object used.
+      # @param script [ScriptView] The batch script used.
+      # @param opts [Hash] The options used to construct a session.
+      def initialize(job, script, opts = {})
+        @job = job
+        @script = script
+      end
+
+      # Submit the VNC job to the defined batch server.
+      #
+      # @param opts [Hash] The options used in job submission.
+      # @option opts [Hash] :headers The headers for the PBS job.
+      # @option opts [Hash] :resources The resources for the PBS job.
+      # @option opts [Hash] :envvars The environment variables for the PBS job.
+      # @return [Session] the session object
+      def submit(opts = {})
+        script.valid? # check if script is valid (can raise errors here)
+
+        h = opts.fetch(:headers, {})
+        r = opts.fetch(:resources, {})
+        e = opts.fetch(:envvars, {})
+
+        # Create tcp listen server if requested
+        listen_server = _create_listen_server(e) if script.tcp_server?
+
+        job.submit(
+          string: script.render,
+          headers: _get_headers(h),
+          resources: _get_resources(r),
+          envvars: _get_envvars(e),
+          qsub: true
+        )
+
+        _write_listen_conn_info(listen_server) if script.tcp_server?
+
+        self
+      end
+
+      # The connection information file.
+      #
+      # @return [String] path to connection information file for this session
+      def conn_file
+        "#{script.outdir}/#{job.id}.conn"
+      end
+
+
+      private
+
+      # The hash of PBS header attributes for the job.
+      def _get_headers(headers)
+        h = {
+          PBS::ATTR[:N] => "VNC_Job",
+          PBS::ATTR[:o] => "#{script.outdir}/$PBS_JOBID.output",
+          PBS::ATTR[:j] => "oe",
+          PBS::ATTR[:S] => "/bin/bash",
+        }.merge headers
+        h[PBS::ATTR[:N]] = "#{ENV['APP_TOKEN']}/#{h[PBS::ATTR[:N]]}" if ENV['APP_TOKEN']
+
+        # add first charged account group as default account
+        account = Process.groups.map {|g| Etc.getgrgid(g).name}.grep(/^P./).first
+        h[PBS::ATTR[:A]] ||= account if account
+
+        h
+      end
+
+      # The hash of PBS resources requested for the job.
+      def _get_resources(resources)
+        {
+          :nodes => "1:ppn=1:#{script.cluster}",
+          :walltime => "24:00:00",
+        }.merge resources
+      end
+
+      # The hash of environment variables passed to the job.
+      def _get_envvars(envvars)
+        {
+        }.merge envvars
+      end
+
+      # Create a tcp listen server and set appropriate environment variables.
+      # for batch script to phone home
+      def _create_listen_server(envvars)
+        listen_server = create_listen
+        _, port, host, _ = listen_server.addr(:hostname)
+        envvars.merge! LISTEN_HOST: host, LISTEN_PORT: port
+        listen_server
+      end
+
+      # Write connection information from a TCP listening server.
+      def _write_listen_conn_info(server)
+        # Wait until VNC conn info is created by PBS batch script
+        # Timeout after 60 seconds if no info is sent
+        Timeout::timeout(60) do
+          File.open(conn_file, 'w', 0600) { |f| f.puts read_listen(server: server) }
+        end
+      end
+    end
+  end
+end

data/lib/osc/vnc/version.rb

@@ -0,0 +1,6 @@
+module OSC
+  module VNC
+    # The current version of osc-vnc.
+    VERSION = "1.0.6"
+  end
+end

data/lib/osc/vnc.rb

@@ -0,0 +1,23 @@
+require 'pbs'
+
+# The namespace used for OSC gems.
+module OSC
+  # The main namespace for osc-vnc. Provides the ability to submit and read back
+  # the connection information for a VNC job on the OSC clusters.
+  module VNC
+    # Path to configuration yml files.
+    CONFIG_PATH = File.dirname(__FILE__) + "/../../config"
+
+    # Patch to PBS script template.
+    SCRIPT_TEMPLATE_PATH = File.dirname(__FILE__) + "/../../templates/script"
+
+    # Path to different connection information view templates.
+    CONN_TEMPLATE_PATH = File.dirname(__FILE__) + "/../../templates/conn"
+  end
+end
+
+require_relative 'vnc/version'
+require_relative 'vnc/error'
+require_relative 'vnc/session'
+require_relative 'vnc/scriptview'
+require_relative 'vnc/connview'

data/osc-vnc.gemspec

@@ -0,0 +1,26 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'osc/vnc/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "osc-vnc"
+  spec.version       = OSC::VNC::VERSION
+  spec.platform      = Gem::Platform::RUBY
+  spec.authors       = ["Jeremy Nicklas"]
+  spec.email         = ["jnicklas@osc.edu"]
+  spec.summary       = %q{Library to create VNC jobs with HPC resources (OSC specific)}
+  spec.description   = %q{This library submits VNC jobs to the Oxymoron cluster as well as obtains the connection information required for a connection to take place with a VNC client. (OSC specific)}
+  spec.homepage      = "https://github.com/OSC/osc-vnc"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency "pbs", "~> 1.0"
+  spec.add_runtime_dependency "mustache", "~> 0.99", ">= 0.99.5"
+  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "rake", "~> 10.0"
+end

data/templates/conn/awesim.mustache

@@ -0,0 +1 @@
+awesim://{{#password}}:{{password}}@{{/password}}{{host}}:{{port}}

data/templates/conn/jnlp.mustache

@@ -0,0 +1,49 @@
+{{^ssh_tunnel?}}
+<?xml version="1.0" encoding="UTF-8"?>
+<jnlp spec="1.0+" codebase="http://mirror.osc.edu/mirror" >
+  <information>
+      <title>TightVnc Viewer</title>
+      <vendor>GlavSoft LLC.</vendor>
+      <offline-allowed/>
+  </information>
+  <security>
+      <all-permissions/>
+  </security>
+  <resources>
+      <j2se version="1.6+"/>
+      <jar href="TightVncViewerSigned.jar" main="true"/>
+  </resources>
+  <application-desc main-class="com.tightvnc.vncviewer.VncViewer">
+      <argument>HOST</argument>
+      <argument>{{host}}</argument>
+      <argument>PORT</argument>
+      <argument>{{port}}</argument>
+      <argument>PASSWORD</argument>
+      <argument>{{password}}</argument>
+  </application-desc>
+</jnlp>
+{{/ssh_tunnel?}}
+{{#ssh_tunnel?}}
+<?xml version="1.0" encoding="UTF-8"?>
+<jnlp spec="1.0+" codebase="http://mirror.osc.edu/mirror" >
+  <information>
+    <title>TightVnc Viewer</title>
+      <vendor>GlavSoft LLC.</vendor>
+      <offline-allowed/>
+  </information>
+  <security>
+   <all-permissions/>
+  </security>
+  <resources>
+    <j2se version="1.6+"/>
+    <jar href="tightvnc-jviewer.jar" main="true"/>
+  </resources>
+  <application-desc main-class="com.glavsoft.viewer.Viewer">
+    <argument>-host={{host}}</argument>
+    <argument>-port={{port}}</argument>
+    <argument>-password={{password}}</argument>
+    <argument>-sshhost={{sshhost}}</argument>
+    <argument>-sshuser={{sshuser}}</argument>
+  </application-desc>
+</jnlp>
+{{/ssh_tunnel?}}

data/templates/conn/novnc.mustache

@@ -0,0 +1 @@
+/rnode/{{host}}/{{web_port}}/vnc_auto.html?password={{#view_only}}{{spassword}}{{/view_only}}{{^view_only}}{{password}}{{/view_only}}&path=rnode/{{host}}/{{web_port}}/websockify

data/templates/conn/osxvnc.mustache

@@ -0,0 +1 @@
+ssh -f -N -L {{localport}}:{{host}}:{{port}} {{sshuser}}@{{sshhost}} && open vnc://:{{password}}@localhost:{{localport}}

data/templates/conn/terminal.mustache

@@ -0,0 +1 @@
+ssh -L {{localport}}:{{host}}:{{port}} {{sshuser}}@{{sshhost}}

data/templates/conn/txt.mustache

@@ -0,0 +1,4 @@
+Host: {{host}}
+Port: {{port}}
+Pass: {{password}}
+Display: {{display_port}}

data/templates/conn/vnc.mustache

@@ -0,0 +1 @@
+vnc://:{{password}}@localhost:{{localport}}

data/templates/conn/yaml.mustache

@@ -0,0 +1,6 @@
+host: {{host}}
+port: {{port}}
+pass: {{password}}
+display: {{display_port}}
+sshuser: {{sshuser}}
+sshhost: {{sshhost}}

data/templates/script/_port_helpers.mustache

@@ -0,0 +1,28 @@
+# Generate random integer in range [$1..$2)
+function random () {
+    local MIN=$1
+    local MAX=$2
+    local RANGE=$((MAX - MIN))
+    local RAND=$(od -An -N4 -t uL /dev/urandom | tr -d " ")
+    local RAND_MAX=$((2**(4*8)))
+
+    local FLOAT=$(echo "$RANGE * ($RAND / $RAND_MAX) + $MIN" | bc -l)
+
+    echo $(echo "$FLOAT / 1" | bc)
+}
+
+# Check if port is used
+function used_ports () {
+    local PORT=$1
+
+    echo $(netstat -lnt | awk -v port=$PORT '$6 == "LISTEN" && $4 ~ ":"port"$"' | wc -l)
+}
+
+# Get available port
+function get_port () {
+  local PORT=$(random 2000 65535)
+  while [[ $(used_ports ${PORT}) -ne 0 ]]; do
+      PORT=$(random 2000 65535)
+  done
+  echo ${PORT}
+}

data/templates/script/_setup_env.mustache

@@ -0,0 +1,7 @@
+export VNC_LOGFILE={{outdir}}/${PBS_JOBID}.log
+{{#tcp_server?}}
+export VNC_OUTFILE=$(mktemp)
+{{/tcp_server?}}
+{{^tcp_server?}}
+export VNC_OUTFILE={{outdir}}/${PBS_JOBID}.conn
+{{/tcp_server?}}

data/templates/script/_ssh_tunnel.mustache

@@ -0,0 +1,28 @@
+{{#ssh_tunnel?}}
+HOST=$(hostname)
+{{/ssh_tunnel?}}
+{{^ssh_tunnel?}}
+# Get correct host
+#
+#    opt2647.ten.osc.edu => vis033.osc.edu
+#    opt2648.ten.osc.edu => vis034.osc.edu
+#    opt2649.ten.osc.edu => vis035.osc.edu
+#    opt2650.ten.osc.edu => vis036.osc.edu
+#    n0691.ten.osc.edu   => oak-vis001.osc.edu
+#    n0692.ten.osc.edu   => oak-vis002.osc.edu
+#
+# 1. we grep the output of /sbin/ip a s to 192.148.0.0/16 to get 192.148.248.70/24, getting the first match only to ignore 192.148.248.255
+#
+#    7: eth0.70@eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP
+#        inet 192.148.248.75/24 brd 192.148.248.255 scope global eth0.70
+#
+# 2. we use this as argument to dig i.e. dig +short -x 192.148.248.70
+#
+#     vis033.osc.edu.
+#
+# 3. we get rid of the ending .
+function public_hostname {
+    /sbin/ip a s to 192.148.0.0/16 | egrep -o '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}' | head -n1 | xargs dig +short -x | sed 's/\.$//'
+}
+HOST=$(public_hostname)
+{{/ssh_tunnel?}}

data/templates/script/_start_vncserver.mustache

@@ -0,0 +1,38 @@
+# Start up vnc server and create output
+# If two servers start simultaneously one will fail, loop until it succeeds
+i=0
+while [[ -z ${VNC_PID} && ${i} -lt 30 ]]; do
+    # Clean up any failed vnc sessions that didn't clean themselves up
+    vncserver -list | awk '/^:/{display[$1] = $2} END{for(key in display) { if( system( "[ ! -e /proc/" display[key] " ]" ) == 0 ) { system( "vncserver -kill " key ) } }}'
+    # Run the vncserver
+    VNC_OUT=$(vncserver \
+        -name {{name}} \
+        -geometry {{geom}} \
+        -dpi {{dpi}} \
+        -fp {{fonts}} \
+        -{{^otp?}}no{{/otp?}}otp \
+        {{#vncauth?}}-rfbauth ${VNC_PASSFILE}{{/vncauth?}}{{^vncauth?}}-novncauth{{/vncauth?}} \
+        -{{^pam?}}no{{/pam?}}pam \
+        -nohttpd \
+        -idletimeout {{idletimeout}} \
+        -log ${VNC_LOGFILE} \
+        -noxstartup \
+        {{{extra_args}}} 2>&1)
+    VNC_PID=$(pgrep -s 0 Xvnc)  # Get process id
+    # VNC couldn't find display so died
+    if [[ -z ${VNC_PID} ]]; then
+        sleep 0.$(($RANDOM % 100))s # Sleep between 0-1 seconds
+    fi
+    # Sometimes VNC will hang if it fails to find working display
+    if [[ ! -z ${VNC_PID} && ${VNC_OUT} =~ "Fatal server error" ]]; then
+        kill -9 ${VNC_PID}
+        unset VNC_PID
+    fi
+    i=$((i+1))  # increment counter
+done
+echo "${VNC_OUT}"
+
+# Make sure it is running otherwise exit with error code 1
+if [[ -z ${VNC_PID} ]]; then
+    exit 1
+fi

data/templates/script/_tcp_server.mustache

@@ -0,0 +1,6 @@
+{{#tcp_server?}}
+# Send connection information over a TCP connection
+# The server information should have been passed as environment variables
+nc ${LISTEN_HOST} ${LISTEN_PORT} < ${VNC_OUTFILE}
+rm -f ${VNC_OUTFILE}
+{{/tcp_server?}}

data/templates/script/server.mustache

@@ -0,0 +1,46 @@
+# Set some environment variables used by this script
+{{> _setup_env}}
+
+{{#xinit}}
+# Port helpers
+{{> _port_helpers}}
+
+# Run developer supplied xinit script
+source {{xinit}}
+{{/xinit}}
+
+# Function that creates a connection information file
+function create_connfile {
+    echo "Host: ${HOST}" > ${VNC_OUTFILE}
+    echo "Port: ${HOST_PORT}" >> ${VNC_OUTFILE}
+    echo "Password: ${PASSWORD}" >> ${VNC_OUTFILE}
+    chmod 600 ${VNC_OUTFILE}
+}
+
+# Clean up script run when "vncserver" finishes or dies prematurely
+# Note: must be defined after `module load`
+function clean_up {
+    echo "Exiting..."
+    {{{xlogout}}}
+}
+trap clean_up TERM EXIT
+
+# Create logfile
+> ${VNC_LOGFILE}
+
+# Run the xstartup script on proper vncserver display
+{{xstartup}} >> ${VNC_LOGFILE} 2>&1 &
+SCRIPT_PID=$(pgrep -s 0 -f {{xstartup}})
+
+{{> _ssh_tunnel}}
+
+# Create the connection file
+create_connfile
+
+{{> _tcp_server}}
+
+# Wait for main process to finish
+while [ -e /proc/${SCRIPT_PID} ]; do sleep 0.1; done
+
+# Exit cleanly
+exit 0

data/templates/script/vnc.mustache

@@ -0,0 +1,86 @@
+# Set some environment variables used by this script
+{{> _setup_env}}
+export VNC_PASSFILE={{outdir}}/${PBS_JOBID}.pass
+
+# Use turbovnc
+{{{load_turbovnc}}}
+
+{{#vncauth?}}
+# Allow the ability to dynamically change VNC password
+function change_passwd {
+    # Generate password for VNC
+    PASSWORD=$(mkpasswd -s 0 -l 8)  # passwords truncated to 8 letters
+    SPASSWORD=${SPASSWORD:-$(mkpasswd -s 0 -l 8)}
+    printf "${PASSWORD}\n${SPASSWORD}" | vncpasswd -f > ${VNC_PASSFILE}
+    chmod 600 ${VNC_PASSFILE}
+}
+change_passwd
+{{/vncauth?}}
+
+# Function that creates a connection information file
+function create_connfile {
+    echo "Host: ${HOST}" > ${VNC_OUTFILE}
+    echo "Port: ${HOST_PORT}" >> ${VNC_OUTFILE}
+    echo "Password: ${PASSWORD}" >> ${VNC_OUTFILE}
+    echo "SPassword: ${SPASSWORD}" >> ${VNC_OUTFILE}
+    echo "Display_Port: ${DISPLAY_PORT}" >> ${VNC_OUTFILE}
+    echo "Web_Port: ${WEB_PORT}" >> ${VNC_OUTFILE}
+    chmod 600 ${VNC_OUTFILE}
+}
+
+# Clean up script run when "vncserver" finishes or dies prematurely
+# Note: must be defined after `module load`
+function clean_up {
+    echo "Exiting..."
+    vncserver -kill :${DISPLAY_PORT}
+    {{{xlogout}}}
+}
+trap clean_up TERM EXIT
+
+{{> _start_vncserver}}
+
+# Parse output for ports
+DISPLAY_PORT=$(echo "${VNC_OUT}" | awk -F':' '/^Desktop/{print $NF}')
+HOST_PORT=$((5900+DISPLAY_PORT))
+
+{{#otp?}}
+# Parse output for password
+PASSWORD=$(echo "${VNC_OUT}" | awk '/^Full/{print $NF}')
+{{/otp?}}
+
+# Run the xstartup script on proper vncserver display
+DISPLAY=:${DISPLAY_PORT} {{xstartup}} >> ${VNC_LOGFILE} 2>&1 &
+SCRIPT_PID=$(pgrep -s 0 -f {{xstartup}})
+
+{{> _ssh_tunnel}}
+
+{{#novnc?}}
+# Port helpers
+{{> _port_helpers}}
+
+# Launch noVNC on available port
+export WEB_PORT=$(get_port)
+{{{novnc_launcher}}} --vnc localhost:${HOST_PORT} --listen ${WEB_PORT} >> ${VNC_LOGFILE} 2>&1 &
+{{/novnc?}}
+
+{{#vncauth?}}
+# Check for authenticated user connections to VNC server then change password
+while read -r line; do
+    if [[ ${line} =~ "Full-control authentication enabled for" ]]; then
+        change_passwd
+        create_connfile
+        echo "Changing VNC password..." >> ${VNC_LOGFILE}
+    fi
+done < <(tail -f ${VNC_LOGFILE}) &
+{{/vncauth?}}
+
+# Create the connection file
+create_connfile
+
+{{> _tcp_server}}
+
+# Wait for main process to finish
+while [ -e /proc/${VNC_PID} ]; do sleep 0.1; done
+
+# Exit cleanly
+exit 0