easel-dashboard 0.4.3 → 0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 15a1dde769e8fa78c0bf23bb6de115bb8f915f0192a672333f7b6d8e2093b2cd
-  data.tar.gz: 1c95d1f20659325da69b2fea9454ce503e278f078b4b6fab3445ad5389bfa67b
+  metadata.gz: 45b20ce9712ac3e87e83e527b03665b9c64a9b707f571cfcbbd1c0989df494e8
+  data.tar.gz: bebb6eeafa225d2a05401fea5f8f8ec7e63674de617a811ec8b6cb2cf7a56640
 SHA512:
-  metadata.gz: 4be4da518caa592bdff010e27ca75d933af403b96958c735f0f33b7502070ccc5f5cc3b5d8df39c1210fca381233559fc594129b7255b3a543fe8eca622b3a98
-  data.tar.gz: 9090eafe766e57975307af322120047931e733534cb0c855dc62baa80fe320477c7001da1e3f5792c363f96756a059ed22a315cb2ee000b3002032413a7aef5a
+  metadata.gz: c50fd692b5fcd3826cc6b3ac80aa400e1319501303f951b576c9c4ada4fe43990a383ed7f84e70d2808363a77dd04e1f07e6b0a2218608309bbed2681a42382b
+  data.tar.gz: d1353d563bf9b9306ebbf5bb69e7a4b401558b4be77010320b7b734808e74124bdbfaecd12890a61a400a2432ee5d16536c909fa9c4c6f2d13885a488465f788

data/bin/easel

@@ -1,4 +1,3 @@
 #!/usr/bin/env ruby
 require 'easel'
-
 launch

data/lib/easel/build_pages.rb

@@ -29,20 +29,55 @@ def build_app
   page
 end
 
+# build_js
+#
+#
+def build_js
+  js_erb = File.new("#{File.dirname(__FILE__)}/../html/controller.js.erb").read
+  page = ERB.new(js_erb).result()
 
-def build_error code
-  error_erb = File.new("#{File.dirname(__FILE__)}/../html/error.html.erb").read
-  page = ERB.new(error_erb).result(binding)
+  "HTTP/1.1 200 OK\r\n" +
+  "Content-Type: text/javascript; charset=UTF-8\r\n" +
+  "Content-Length: #{page.bytesize}\r\n" +
+  "Connection: close\r\n" +
+  "\r\n" +
+  page
+end
 
-  "HTTP/1.1 #{code} #{@code_names[code]}\r\n" +
-  "Content-Type: text/html; charset=UTF-8\r\n" +
+
+# return_js
+#
+#
+def return_js file
+
+  page = File.new("#{File.dirname(__FILE__)}/../html/#{file}").read
+
+  "HTTP/1.1 200 OK\r\n" +
+  "Content-Type: text/javascript; charset=UTF-8\r\n" +
   "Content-Length: #{page.bytesize}\r\n" +
   "Connection: close\r\n" +
   "\r\n" +
   page
 end
 
+# return_html
+#
+#
+def return_html file
+
+  page = File.new("#{File.dirname(__FILE__)}/../html/#{file}").read
 
+  "HTTP/1.1 200 OK\r\n" +
+  "Content-Type: text/html; charset=UTF-8\r\n" +
+  "Content-Length: #{page.bytesize}\r\n" +
+  "Connection: close\r\n" +
+  "\r\n" +
+  page
+end
+
+# build_css
+#
+#
 def build_css
   error_erb = File.new("#{File.dirname(__FILE__)}/../html/app.css.erb").read
   css = ERB.new(error_erb).result(binding)
@@ -54,3 +89,19 @@ def build_css
   "\r\n" +
   css
 end
+
+
+# build_error
+#
+#
+def build_error code
+  error_erb = File.new("#{File.dirname(__FILE__)}/../html/error.html.erb").read
+  page = ERB.new(error_erb).result(binding)
+
+  "HTTP/1.1 #{code} #{@code_names[code]}\r\n" +
+  "Content-Type: text/html; charset=UTF-8\r\n" +
+  "Content-Length: #{page.bytesize}\r\n" +
+  "Connection: close\r\n" +
+  "\r\n" +
+  page
+end

data/lib/easel/configuration.rb

@@ -11,11 +11,13 @@
 # Global Variables
 $config = {
   logging: 2,             # 0=Fatal, 1=Error, 2=Warning, 3=Info
-  port: 4200,             # Default port
-  hostname: 'localhost',  # Default hostname
-  log_file: STDOUT,       # Default logging to STDOUT
-  title: 'Easel - Your Custom Dashboard',
-  colours: {
+  port: 4200,             # Port # to bind Easel to
+  hostname: 'localhost',  # Hostname to accept. "" Accepts all connections.
+  log_file: STDOUT,       # Where to write the logs to.
+  title: 'Easel - Your Custom Dashboard', # The title of the dashboard webpage.
+  header_logo: '',       # TODO: have nil mean default to Easel's, otherwise put in src=""
+  header_title: '<a class="on-hover-secondary" href="https://easeldashboard.com">Easel Dashboard</a>', # TODO: the same as the logo line basically.
+  colours: {                    # The RGB values for the dashboard. TODO: accept hsl in HTML format.
     surface:        '#222222',
     background:     '#000000',
     primary:        '#7DF9FF',
@@ -28,7 +30,7 @@ $config = {
     stdout_colour:  '#ffffff',
     stderr_colour:  '#00FF00'
   },
-  commands: [
+  commands: [  # A list of commands to allow the user to run via Easel.
       {
         name: 'Test 1',
         cmd:  'echo "this is the output of Test 1"',
@@ -39,5 +41,51 @@ $config = {
         cmd:  'echo "this is the output of Test 2"',
         desc: 'Simple output test #2'
       }
+  ],
+  collect_data_period: 15,
+  dashboards: [ # A list of dashboards to display
+    {
+      name: "Server Status",
+      desc: "Basic server status.",
+      id: "A", # TODO: dashboard and element IDs should be automatically added.
+      elements: [
+        {
+          name: "CPU Load",
+          type: "time-series",  # uses Time.now.strftime("%H:%M") as x asix.
+          data: [
+            {
+              cmd:   "uptime",
+              name:  "1min Average",
+              regex: "average: (\\d+.\\d+)"
+            },
+            {
+              cmd:   "uptime",
+              name:  "5min Average",
+              regex: ", (\\d+.\\d+),"
+            },{
+              cmd:   "uptime",
+              name:  "15min Average",
+              regex: ", (\\d+.\\d+)\\n"
+            }
+          ]
+        },
+        {
+          name: "Memory",
+          type: "time-series",  # uses Time.now.strftime("%H:%M") as x asix.
+          data: [
+            {
+              cmd:   "free",
+              name:  "Total Memory",
+              regex: "Mem:\\W+(\\d+)"
+            },
+            {
+              cmd:   "free",
+              name:  "Free Memory",
+              regex: "Mem:\\W+\\d+\\W+(\\d+)"
+            }
+          ]
+        }
+      ]
+    }
   ]
 }

data/lib/easel/controller.rb

@@ -0,0 +1,21 @@
+#!/snap/bin/ruby
+#
+# Author: Eric Power
+
+# Imports
+require_relative './configuration'
+
+
+def launch_easel config
+
+  # r_server_data = Ractor.new config do
+  #   'this is a message' # TODO: Implement this.
+  # end
+
+  tcp_listener = Ractor.new(config) do |config|
+    launch_server config
+  end
+
+  tcp_listener.take
+
+end

data/lib/easel/data_gathering.rb

@@ -0,0 +1,144 @@
+#!/snap/bin/ruby
+#
+# Author: Eric Power
+#
+# Description:
+#     Collects information about the system to display via the Easel dashboard.
+#     Uses semaphores and mutexes to provide atomic reads and writes. Allows a
+#     single writer at a time, but any number of readers can read at once.
+#
+#     WARNING: this feature currently holds all information in memory, so don't
+#     turn this on if you're expecting to leave Easel running for a long time.
+#
+#     Plan for adressing this:
+#       - Move 'historical' data to a proper datastore (eg. a SQLite database)
+#       - Have only the most recent readings in @collected_data, to allow the
+#         websocket threads to pull the up to date data from there.
+#       - Eventually allow the clients to query the historial datastore, so it's
+#         likely that the best bet is to have the database store the data with
+#         the time as the dense index.
+
+
+# Imports
+require 'thread'
+require 'concurrent'
+
+# Key Variables
+@collected_data = {}
+@writers_semaphore = Concurrent::Semaphore.new(0)  # Used to count currently active writers
+@readers_semaphore = Concurrent::Semaphore.new(0)  # Used to count currently active readers
+@join_mutex = Mutex.new
+
+# launch_data_collection
+#
+# Launch a background thread to start collecting system info in the background.
+def launch_data_collection
+  Thread.new do
+    loop do
+      collect_data
+      sleep $config[:collect_data_period]
+    end
+  end
+end
+
+
+# collect_data
+#
+# Collects information on the current state of the system, and writes it to
+# @collected_data.
+def collect_data
+  new_data = {}
+
+  # TODO: Check which element type something is to figure out how to handle it.
+
+  log_info "Collecting data."
+  $config[:dashboards].each{ |dashboard|
+
+    new_data[dashboard[:id]] = {}
+    dashboard[:elements].each_with_index{ |element, e_index|
+      new_data[dashboard[:id]][e_index] = {}
+
+      element[:data].each_with_index{ |data, index|
+        output = `#{data[:cmd]}`
+        log_info "Ran `#{data[:cmd]}`, got: #{output}"
+        begin
+          value = output.match(/#{data[:regex]}/)[1]
+        rescue NoMethodError => e
+          log_error "Data failed to be parsed. Regex: /#{data[:regex]}/ -- Output: #{output}"
+        end
+        new_data[dashboard[:id]][e_index][index] = [
+          Time.new.strftime("%H:%M:%S"),
+          value
+        ]
+      }
+    }
+  }
+  write_data new_data
+end
+
+
+# write_data
+#
+# Writes the data collected to @collected_data, and handles the Readers/Writer
+# problem by using semaphores and a mutex.
+def write_data data
+
+  joined = false
+  until joined
+    @join_mutex.synchronize {
+      if @readers_semaphore.available_permits == 0 and @writers_semaphore.available_permits == 0
+        @writers_semaphore.release 1  # Increment @writers_semaphore
+        joined = true
+      end
+    }
+    sleep 0.05  # Wait 50ms to give another thread time to lock the @join_mutex.
+  end
+
+  # Write Data
+  data.each_key { |key|
+    case key
+    when :load
+      @collected_data[:load] = [] if @collected_data[:load].nil?
+      @collected_data[:load] << data[:load]
+    else
+      @collected_data[key] = data[key]
+    end
+  }
+
+  # Log if @collected_data has gotten too large.
+  if not @collected_data[:load].nil? and @collected_data[:load].length > 240
+    if @collected_data[:load].length > 1000
+      log_error "Easel dashboard has run collect_data more than a 1000 times. Memory size likely to be large."
+    else
+      log_warning "Easel dashboard starting to take up a lot of memory due to data_collection being turned on."
+    end
+  end
+
+  @writers_semaphore.acquire 1  # Decrement @writers_semaphore
+end
+
+
+# read_data
+#
+# Reads (copies) @collected_data, and handles the Readers/Writer
+# problem by using semaphores and a mutex. Returns a copy
+def read_data
+
+  joined = false
+  until joined
+    @join_mutex.synchronize {
+      if @writers_semaphore.available_permits == 0
+        @readers_semaphore.release 1  # Increment @readers_semaphore
+        # TODO: likely need to release the mutex.
+        joined = true
+      end
+    }
+    sleep 0.05 unless joined  # Wait 50ms to give another thread time to lock the @join_mutex.
+  end
+
+  # Read Data
+  data = @collected_data.dup
+
+  @readers_semaphore.acquire 1  # Decrement @readers_semaphore
+  data
+end

data/lib/easel/server.rb

@@ -7,8 +7,9 @@
 
 # Imports
 require 'socket'
-require_relative './build_pages.rb'
+require_relative './build_pages'
 require_relative './websocket'
+require_relative './data_gathering'
 
 
 def launch_server
@@ -20,6 +21,9 @@ def launch_server
     log_fatal "Server could not start. Error message: #{e}"
   end
 
+  # Lauch data collection if turned on.
+  launch_data_collection unless $config[:collect_data_period] == 0
+
   Thread.abort_on_exception = true
 
   # Main Loop
@@ -29,10 +33,10 @@ def launch_server
         handle_request client
     end
   }
-
-  # Handle shutting down.
-  rescue Interrupt
+  rescue Interrupt # Handle shutting down.
     log_info "Interrupt received, server shutting down..."
+  rescue Exception => e
+    log_error "Unexpected error occured and closed client connection. Error: #{e}"
   end
 end
 
@@ -55,7 +59,6 @@ def handle_request socket
     # TODO: Deal with HEAD request. https://www.w3.org/Protocols/rfc2616/rfc2616-sec9.html#sec9.4
   else
     # TODO: respond with an appropriate error.
-    response "I don't understand what you sent - go away."
     socket.print "HTTP/1.1 200 OK\r\n" +
                  "Content-Type: text/plain\r\n" +
                  "Content-Length: #{response.bytesize}\r\n" +
@@ -78,9 +81,22 @@ def handle_get(socket, request)
   when "/", "/index.html"
     socket.print build_app
     socket.close
+  when "/test.html" # TODO: Remove this!
+    socket.print return_html "test.html"
+    socket.close
   when "/app.css"
     socket.print build_css
     socket.close
+  when "/controller.js"
+    log_info "building controller"
+    socket.print build_js
+    socket.close
+  when "/dashboardElements.js"
+    socket.print return_js 'dashboardElements.js'
+    socket.close
+  when "/createComponents.js"
+    socket.print return_js 'createComponents.js'
+    socket.close
   else
     socket.print build_error 404
     socket.close
@@ -102,7 +118,6 @@ def read_HTTP_message socket
     end
   end
 
-
   request = {fields: {}}
   (request[:method], request[:url], request[:protocol]) = message[0].split(" ")
 

data/lib/easel/websocket.rb

@@ -9,14 +9,66 @@
 #
 # Note: Websocket code is based on the code provided in the following article:
 # https://www.honeybadger.io/blog/building-a-simple-websockets-server-from-scratch-in-ruby/
+#
+# Overview of the websocket protocol:
+#     Communication over the websocket is split into two message types:
+#
+#       1. Command specific messages
+#             These messages are sent from either the client or the server, and
+#             are associated with a particular command via an ID. This ID is set
+#             by the server when building a page.
+#
+#             The server sends:
+#                 - ID:OUT:XXXXXXX where the contents of XXXX is new content in
+#                   the stdOUT of the command that's associated with ID.
+#                 - ID:ERR:XXXXXXX where the contents of XXXX is new content in
+#                   the stdERR of the command that's associated with ID.
+#                 - ID:FINISHED tells the client that the command associated with
+#                   ID has finished running.
+#                 - ID:CLEAR tells the client that the command associated with ID
+#                   wants to clear the current output. This is useful when
+#                   handling X-term escape sequences (eg. `top` wants a fresh page)
+#             The client sends:
+#                 - ID:RUN requests that the server start running the command
+#                   associated with ID.
+#                 - ID:STOP requests that the server stop running the command
+#                   associated with ID.
+#
+#       2. Dashboard Information
+#             These messages let the client indicate what information it wants to
+#             handle, and let the server send that content to the client either
+#             as a blob (if the information is historical) or as a stream (if
+#             the information is real time). When the client requests data, it
+#             includes a DID. The difference between the DID and an ID that
+#             is associated with a command is that the DID contains a letter
+#             at the start (eg. A12).
+#
+#             DIDs are structured as a letter that represents a dashboard, and
+#             a number that represents the element on the dashboard.
+#
+#             The server sends:
+#                 - DID:Y:XXXXXXXX sends data that should be connected to the
+#                   DID. The Y is either an A (representing "ALL") to say that the
+#                   message is a self-contained data point, or a ratio like (1/2)
+#                   to say its the first of two messages that when combined form
+#                   a data point.
+#
+#             At this point, the client does not send anything related to DIDs.
+#             Options in the future include letting the client request a range of
+#             data, but that will likely wait until the data on the server end
+#             is moved out of memory an on to disk.
+
 
 
 # Imports
-require 'digest'
-require 'open3'
+require 'digest'  # Allows hashing of websocket authentication value
+require 'open3'   # Allows capturing stdout and stderr of system commands
+require 'thread'  # Allows use of mutexes.
+
+require_relative 'data_gathering'
 
 # Key Variables
-MAX_WS_FRAME_SIZE = 50.0
+MAX_WS_FRAME_SIZE = 50.0  # Must be a float number to allow a non-truncated division result.
 
 # run_websocket
 #
@@ -25,6 +77,15 @@ def run_websocket(socket, initial_request)
 
   accept_connection(socket, initial_request[:fields][:Sec_WebSocket_Key][0..-3])
   child_threads = {}
+  send_msg_mutex = Mutex.new # One mutex per websocket to control sending messages.
+
+  Thread.new {  # Periodically update the generic dashboard if set.
+    loop do
+      data = read_data
+      send_msg(socket, send_msg_mutex, nil, "DASH", data)
+      sleep $config[:collect_data_period]
+    end
+  } unless $config[:collect_data_period] == 0
 
   loop {
     msg = receive_msg socket
@@ -36,10 +97,11 @@ def run_websocket(socket, initial_request)
 
       unless child_threads[cmd_id]
         child_threads[cmd_id] = Thread.new do
-          run_command_and_stream(socket, cmd_id)
+          run_command_and_stream(socket, cmd_id, send_msg_mutex)
           child_threads[cmd_id] = nil
         end
       end
+
     when "STOP"
 
       cmd_id = msg.match(/^STOP:(.*)$/)[1].to_i
@@ -59,7 +121,7 @@ end
 # run_command_and_stream
 #
 # Run a command and stream the stdout and stderr through the websocket.
-def run_command_and_stream(socket, cmd_id)
+def run_command_and_stream(socket, cmd_id, send_msg_mutex)
 
   cmd = get_command cmd_id
   if cmd.nil?
@@ -78,9 +140,9 @@ def run_command_and_stream(socket, cmd_id)
           break
         end
         if fd == stdout
-          send_msg(socket, cmd_id, "OUT", resp, )
+          send_msg(socket, send_msg_mutex, cmd_id, "OUT", resp)
         elsif fd == stderr
-          send_msg(socket, cmd_id, "ERR", resp, )
+          send_msg(socket, send_msg_mutex, cmd_id, "ERR", resp)
         else
           raise "Received output from popen3(#{cmd}) that was not via stdout or stderr."
         end
@@ -89,7 +151,7 @@ def run_command_and_stream(socket, cmd_id)
     end
 
     cmd_thread.join
-    send_msg(socket, cmd_id, "FINISHED")
+    send_msg(socket, send_msg_mutex, cmd_id, "FINISHED")
   end
 end
 
@@ -161,47 +223,91 @@ end
 # send_msg
 #
 #
-def send_msg(socket, cmd_id, msg_type, msg=nil)
+def send_msg(socket, send_msg_mutex, cmd_id, msg_type, msg=nil)
 
 
-  # TODO: Figure out the proper frame size (MAX_WS_FRAME_SIZE).
-  def send_frame(socket, fmsg)
-    output = [0b10000001, fmsg.size, fmsg]
-    socket.write output.pack("CCA#{fmsg.size}")
-  end
-
   case msg_type
-  when "OUT", "ERR"
+  when "OUT", "ERR" # See comments at the top of the file to explain this part of the protocol.
     header = "#{cmd_id}:#{msg_type}:"
     if header.length > MAX_WS_FRAME_SIZE
       log_error "Message header '#{msg_type}' is too long. Msg: #{msg}."
     elsif msg.nil?
       log_error "Message of type '#{msg_type}' sent without a message."
     else
-      if msg.length > MAX_WS_FRAME_SIZE - header.length
-        msg_part_len = MAX_WS_FRAME_SIZE - header.length
-        msg_parts = (0..(msg.length-1)/msg_part_len).map{ |i|
-          msg[i*msg_part_len,msg_part_len]
-        }
-        msg_parts.each{ |part|
-          send_frame(socket, header + part)
-        }
-      else
-        send_frame(socket, header + msg)
-      end
+      send_msg_mutex.synchronize {
+        if msg.length > MAX_WS_FRAME_SIZE - header.length
+          msg_part_len = MAX_WS_FRAME_SIZE - header.length
+          msg_parts = (0..(msg.length-1)/msg_part_len).map{ |i|
+            msg[i*msg_part_len,msg_part_len]
+          }
+          msg_parts.each{ |part|
+            send_frame(socket, header + part)
+          }
+        else
+          send_frame(socket, header + msg)
+        end
+      }
     end
 
+  when "DASH"  # See comments at the top of the file to explain this part of the protocol.
+    if msg.nil?
+      log_error "Message of type '#{msg_type}' sent without a message."
+    end
+    msg.each_key { |dash_id|
+      msg[dash_id].each_key { |ele_index|
+        did = "#{dash_id}#{ele_index}"
+        send_msg_mutex.synchronize {
+          msg[dash_id][ele_index].each_key { |key|
+            data_fragment = "#{key}->#{msg[dash_id][ele_index][key]}"
+            if data_fragment.length > MAX_WS_FRAME_SIZE - (did.length + 3)
+              msg_part_len = MAX_WS_FRAME_SIZE - (did.length + 7) # TODO: Handle case where header is longer than DID:XX/XX:
+              msg_parts = (0..(data_fragment.length-1)/msg_part_len).map{ |i|
+                data_fragment[i*msg_part_len,msg_part_len]
+              }
+              msg_parts.each_with_index{ |part, index|
+                header = did + ":#{index + 1}/#{msg_parts.length}:"
+                if header.length > MAX_WS_FRAME_SIZE
+                  log_error "Message header '#{msg_type}' is too long. Data: #{data_fragment}."
+                end
+                send_frame(socket, header + part)
+              }
+            else
+              send_frame(socket, did + ":A:" + data_fragment)
+            end
+          }
+        }
+      }
+    }
+
   when "CLEAR", "FINISHED"
-    to_send = "#{cmd_id}:#{msg_type}"
-    if to_send.length > MAX_WS_FRAME_SIZE
-      log_error "Message of type '#{msg_type}' is too long. Msg: #{to_send}."
-    elsif !msg.nil?
-      log_error "Message of type '#{msg_type}' passed a message. Msg: #{msg}."
-    else
-      send_frame(socket, to_send)
+    if !msg.nil?
+      log_error "Message of type '#{msg_type}' passed an empty message. Msg: #{msg}."
     end
+    to_send = "#{cmd_id}:#{msg_type}"
+    send_msg_mutex.synchronize {
+      if to_send.length > MAX_WS_FRAME_SIZE
+        log_error "Message of type '#{msg_type}' is too long. Msg: #{to_send}."
+      else
+        send_frame(socket, to_send)
+      end
+    }
   else
     log_error "Trying to send a websocket message with unrecognized type: #{msg_type}"
   end
 
 end
+
+# send_frame
+#
+# Sends a message over the websocket. Requires that the message be an appropriate
+# length, and have the right format (eg. checks should be done before calling
+# this function).
+# TODO: Figure out the proper frame size (MAX_WS_FRAME_SIZE).
+def send_frame(socket, msg)
+  output = [0b10000001, msg.size, msg]
+  begin
+    socket.write output.pack("CCA#{msg.size}")
+  rescue IOError
+    log_error "WebSocket is closed. Msg: #{msg}"
+  end
+end