internet_box_logger 0.1.1

data/config/schedule.rb

@@ -0,0 +1,36 @@
+# Use this file to easily define all of your cron jobs.
+#
+# It's helpful, but not entirely necessary to understand cron before proceeding.
+# http://en.wikipedia.org/wiki/Cron
+
+# Example:
+#
+# set :output, "/path/to/my/cron_log.log"
+#
+# every 2.hours do
+#   command "/usr/bin/some_great_command"
+#   runner "MyModel.some_method"
+#   rake "some:great:rake:task"
+# end
+#
+# every 4.days do
+#   runner "AnotherModel.prune_old_records"
+# end
+
+# Learn more: http://github.com/javan/whenever
+
+# Default interval
+# To actually set the default interval, use cron_interval in your application config
+require 'active_support/all'
+
+@interval ||= 1
+
+this_gem_path = File.expand_path('../..', __FILE__)
+set :path, this_gem_path
+set :output, "#{this_gem_path}/log/cron.log"
+
+job_type :ruby_script, 'cd :path && bundle exec ruby bin/:task :output'
+
+every @interval.to_i.minute do
+  ruby_script 'internet_box_logger'
+end

data/internet_box_logger.gemspec

@@ -0,0 +1,31 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'internet_box_logger/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'internet_box_logger'
+  spec.version       = InternetBoxLogger::VERSION
+  spec.authors       = ['Laurent B']
+  spec.email         = ['lbnetid+gh@gmail.com']
+  spec.summary       = %q{Monitor your internet box.}
+  spec.description   = %q{Logs information gathered from your internet box and stores into ElasticSearch for display into Kibana.}
+  spec.homepage      = ''
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'bundler'
+  spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'rspec'
+  spec.add_development_dependency 'pry'
+
+  spec.add_dependency 'activesupport'
+  spec.add_dependency 'whenever'
+  spec.add_dependency 'easy_app_helper'
+  spec.add_dependency 'elasticsearch'
+
+end

data/lib/internet_box_logger/elastic_search.rb

@@ -0,0 +1,61 @@
+module InternetBoxLogger
+  module ElasticSearch
+
+    module Server
+
+      def self.[]
+        EasyAppHelper.config[:elastic_servers] || EasyAppHelper.config[:default_elastic_search]['elastic_servers']
+      end
+
+      def self.local_path
+        EasyAppHelper.config[:elastic_binary] || EasyAppHelper.config[:default_elastic_search]['elastic_binary']
+      end
+
+      def self.local?
+        !remote?
+      end
+
+      def self.manageable?
+        !(remote? || local_path.nil? )
+      end
+
+      def self.remote?
+        # res = true
+        # return nil if Server[].nil? || Server.empty?
+        # Server[].each do |addr|
+        #   if
+        #
+        #   end
+        # end
+        local_path.nil?
+      end
+
+    end
+
+
+
+
+    def elasticsearch_client
+      @elasticsearch_client ||= Elasticsearch::Client.new hosts: Server[], log: EasyAppHelper.config[:debug], reload_connections: true
+    end
+
+    def save
+      internal_representation = []
+      EasyAppHelper.logger.debug 'Saving measurements to ElasticSearch'
+      self.as_es_documents.each do |document|
+        begin
+          EasyAppHelper.logger.debug " - #{document[:body][:name]}"
+          internal_representation << elasticsearch_client.index(**document)
+        rescue => e
+          EasyAppHelper.logger.error e.message
+        end
+      end
+      @internal_es_representation = internal_representation
+      EasyAppHelper.puts_and_logs 'Your box metrics have been indexed into Elastic Search'
+      self
+    rescue
+      EasyAppHelper.logger.error 'Unable to save to ElasticSearch !!'
+    end
+
+  end
+end

data/lib/internet_box_logger/generic_box.rb

@@ -0,0 +1,21 @@
+
+module InternetBoxLogger
+  class GenericBox
+
+    include InternetBoxLogger::ElasticSearch
+
+    def initialize(box_type)
+      box_type = box_type.to_s if box_type.is_a? Symbol
+      box_type = self.class.const_get box_type if box_type.is_a? String
+      box_parser_module = box_type if InternetBoxLogger::Parsers[].include? box_type
+      self.extend box_parser_module
+    end
+
+    def log_box_info
+      save if get_box_data
+    end
+
+
+
+  end
+end

data/lib/internet_box_logger/parsers/freebox_v5_parser.rb

@@ -0,0 +1,127 @@
+require 'open-uri'
+
+module InternetBoxLogger
+  module Parsers
+    module FreeboxV5Parser
+
+      include InternetBoxLogger::Parsers::Utils
+
+      DEFAULT_STATUS_URL = 'http://mafreebox.free.fr/pub/fbx_info.txt'
+
+      EXPECTED_LINES = [
+          /version du firmware\s+(?<firmware_version>[\d\.]+)\s*$/i ,
+          /mode de connection\s+(?<connection_mode>[[:alpha:]]+)\s*$/i ,
+          /temps depuis la mise en route\s+(?<up_time>.*)$/i ,
+          /etat\s+(?<phone_status>[[:alpha:]]+)\s*$/i ,
+          /etat du combiné\s+(?<phone_hanged_up>[[:alpha:]]+)\s*$/i ,
+          /sonnerie\s+(?<phone_ringing>[[:alpha:]]+)\s*$/i ,
+          /etat\s+(?<adsl_status>[[:alpha:]]+)\s*$/i ,
+          /protocole\s+(?<adsl_protocol>[^\s]+)\s*$/i ,
+          /mode\s+(?<adsl_mode>[[:alpha:]]+)\s*$/i ,
+          /débit atm\s+(?<adsl_atm_bandwidth_down>[\d\.]+\s+[^\s]+)\s+(?<adsl_atm_bandwidth_up>[\d\.]+\s+[^\s]+)\s*$/i ,
+          /marge de bruit\s+(?<adsl_noise_margin_down>[\d\.]+\s+[^\s]+)\s+(?<adsl_noise_margin_up>[\d\.]+\s+[^\s]+)\s*$/i ,
+          /atténuation\s+(?<adsl_attenuation_down>[\d\.]+\s+[^\s]+)\s+(?<adsl_attenuation_up>[\d\.]+\s+[^\s]+)\s*$/i ,
+          /fec\s+(?<adsl_fec_down>\d+)\s+(?<adsl_fec_up>\d+)\s*$/i ,
+          /crc\s+(?<adsl_crc_down>\d+)\s+(?<adsl_crc_up>\d+)\s*$/i ,
+          /hec\s+(?<adsl_hec_down>\d+)\s+(?<adsl_hec_up>\d+)\s*$/i ,
+          /etat\s+wifi\s+(?<wifi_state>[[:alpha:]]+)\s*$/i ,
+          /canal\s+(?<wifi_chanel>[\d]+)\s*$/i ,
+          /tat du r[eé]seau\s+(?<wifi_network_state>[[:alpha:]]+)\s*$/i ,
+          /freewifi\s+(?<wifi_freewifi_state>[[:alpha:]]+)\s*$/i ,
+          /freewifi secure\s+(?<wifi_freewifi_secure_state>[[:alpha:]]+)\s*$/i ,
+          /adresse mac freebox\s+(?<network_mac>[\dabcdefABCDEF:]+)\s*$/i ,
+          /\s+(?<network_public_ipv4>[\d\.]+)\s*$/i ,
+          /mode routeur\s+(?<network_router>[[:alpha:]]+)\s*$/i ,
+          /adresse ip privée\s+(?<network_private_ipv4>[\d\.]+)\s*$/i ,
+          /adresse ip dmz\s+(?<network_dmz_ipv4>[\d\.]+)\s*$/i ,
+          /adresse ip freeplayer\s+(?<network_freeplayer_ipv4>[\d\.]+)\s*$/i ,
+          /réponse au ping\s+(?<network_answer_to_ping>[[:alpha:]]+)\s*$/i ,
+          /proxy wake on lan\s+(?<network_wake_on_lan>[[:alpha:]]+)\s*$/i ,
+          /serveur dhcp\s+(?<network_dhcp>[[:alpha:]]+)\s*$/i ,
+          /plage d'adresses dynamiques?\s+(?<network_dhcp_range>[\d\.\s-]+[^\s])\s*$/i ,
+      ]
+
+      FIELD_POST_PROCESSING = {
+          up_time: :to_duration,
+          phone_status: :to_bool,
+          phone_hanged_up: :to_bool,
+          phone_ringing: :to_bool,
+          wifi_chanel: :to_int,
+          wifi_state: :to_bool,
+          wifi_network_state: :to_bool,
+          wifi_freewifi_state: :to_bool,
+          wifi_freewifi_secure_state: :to_bool,
+          network_answer_to_ping: :to_bool,
+          network_wake_on_lan: :to_bool,
+          network_dhcp: :to_bool,
+          network_router: :to_bool,
+          adsl_atm_bandwidth_up: :to_bandwidth,
+          adsl_atm_bandwidth_down: :to_bandwidth,
+          adsl_noise_margin_up: :to_db,
+          adsl_noise_margin_down: :to_db,
+          adsl_attenuation_up: :to_db,
+          adsl_attenuation_down: :to_db,
+          adsl_crc_up: :to_num,
+          adsl_crc_down: :to_num,
+          adsl_fec_up: :to_num,
+          adsl_fec_down: :to_num,
+          adsl_hec_up: :to_num,
+          adsl_hec_down: :to_num,
+      }
+
+      UP_DOWN_REPORTS = {
+          adsl_noise_margin: 'Noise Margin',
+          adsl_atm_bandwidth: 'ATM Bandwidth',
+          adsl_attenuation: 'Attenuation',
+          adsl_crc: 'CRC',
+          adsl_fec: 'FEC',
+          adsl_hec: 'HEC'
+      }
+
+      attr_accessor :raw_data, :attributes
+
+      def get_status_url
+        EasyAppHelper.config[:freebox_alternate_url] ? EasyAppHelper.config[:freebox_alternate_url] : DEFAULT_STATUS_URL
+      end
+
+      def up_down_reports
+        UP_DOWN_REPORTS
+      end
+
+      def get_box_data
+        regexp_list = EXPECTED_LINES.dup
+        current_regexp = nil
+        @raw_data, @attributes = [], {}
+        skip_parsing = false
+        open(get_status_url).readlines.each do |line|
+          @raw_data << line
+          next if skip_parsing
+          begin
+            current_regexp = regexp_list.shift if current_regexp.nil?
+          rescue
+            EasyAppHelper.logger.info 'Got all data. Do not parse the rest of the data.'
+            skip_parsing = true
+          end
+          break if current_regexp.nil?
+          line.encode('utf-8').match current_regexp do |md|
+            md.names.each do |field|
+              EasyAppHelper.logger.info "#{field} => #{md[field]}"
+              @attributes[field.to_sym] = normalize_value(field.to_sym, md)
+              current_regexp = nil
+            end
+          end
+        end
+        # Check if the parsing has been complete
+        regexp_list.empty? ? self : nil
+      end
+
+
+      def normalize_value(field_name, match_data)
+        return match_data[field_name] unless FIELD_POST_PROCESSING[field_name]
+        self.send FIELD_POST_PROCESSING[field_name], field_name, match_data[field_name]
+      end
+
+
+    end
+  end
+end

data/lib/internet_box_logger/parsers/utils.rb

@@ -0,0 +1,116 @@
+module InternetBoxLogger
+  module Parsers
+    module Utils
+
+      CONSIDERED_TRUE = %w(actif active activée activé ok true connectée connecté on décroché 1)
+      CONSIDERED_FALSE = %w(inactif inactive desactivé desactivée deconnecté deconnectée désactivé désactivée déconnecté déconnectée ko false off raccroché 0)
+
+      def to_bandwidth(field_name, value_to_convert)
+        value_to_convert.match /^\s*(?<val>[\d\.]+)\s+(?<unit>[kKMmGg])b\/s/ do |md|
+          mult = case md[:unit]
+                   when 'k', 'K' then
+                     1024
+                   when 'm', 'M' then
+                     1024 * 1024
+                   when 'g', 'G' then
+                     1024 * 1024
+                 end
+          return md[:val].to_f * mult
+        end
+
+        EasyAppHelper.logger.warn "Cannot convert #{value_to_convert.inspect} to time duration (integer) for field #{field_name} !"
+        nil
+      end
+
+      def to_duration(field_name, value_to_convert)
+        # 9 jours, 22 heures, 42 minutes
+        value_to_convert.match /(?<days>\d+)\s*jours?,\s*(?<hours>\d+)\s*heures?,\s*(?<minutes>\d+)\s*minutes?/i do |md|
+          d = md[:days].nil? ? 0 : md[:days].to_i
+          h = md[:hours].nil? ? 0 : md[:hours].to_i
+          m = md[:minutes].nil? ? 0 : md[:minutes].to_i
+          return d * 86400 + h * 3600 + m * 60
+        end
+        EasyAppHelper.logger.warn "Cannot convert #{value_to_convert.inspect} to time duration (integer) for field #{field_name} !"
+        nil
+      end
+
+      def to_bool(field_name, value_to_convert)
+        CONSIDERED_TRUE.each do |val|
+          return true if value_to_convert.match /^#{val}$/i
+        end
+        CONSIDERED_FALSE.each do |val|
+          return false if value_to_convert.match /^#{val}$/i
+        end
+        EasyAppHelper.logger.warn "Cannot convert #{value_to_convert.inspect} to boolean for field #{field_name} !"
+        nil
+      end
+
+      def to_db(field_name, value_to_convert)
+        value_to_convert.match /^(?<num_value>.+) db$/i do |md|
+          return to_num(field_name, md[:num_value])
+        end
+        EasyAppHelper.logger.warn "Cannot convert #{value_to_convert.inspect} to db for field #{field_name} !"
+        nil
+      end
+
+      def to_num(field_name, value_to_convert)
+        value_to_convert.match /^(?<num_value>[[:digit:]\.\s,]+)$/i do |md|
+          return md[:num_value].to_f
+        end
+        EasyAppHelper.logger.warn "Cannot convert #{value_to_convert.inspect} to num for field #{field_name} !"
+        nil
+      end
+
+      def to_int(field_name, value_to_convert)
+        value_to_convert.match /^(?<num_value>[[:digit:]]+)$/i do |md|
+          return md[:num_value].to_i
+        end
+        EasyAppHelper.logger.warn "Cannot convert #{value_to_convert.inspect} to integer for field #{field_name} !"
+        nil
+      end
+
+      def as_es_documents(created_at=Time.now)
+        res = []
+        self.up_down_reports.each_pair do |measurement, name|
+          %w(up down).each do |measurement_type|
+            data_name = "#{measurement}_#{measurement_type}"
+            es_object = {
+                index: "#{self.class.name.underscore.tr('/', '_')}_#{measurement}",
+                type: measurement_type
+            }
+            data = {
+                created_at: created_at,
+                name: data_name,
+                description: name,
+                value: attributes[data_name.to_sym]
+            }
+            es_object[:body] = data
+            res << es_object
+          end
+        end
+        generic_info = {}
+
+        attributes.each do |attr_name, content|
+          # Tries to remove data that are up/down measurements already covered by previous collection
+          data_key = attr_name.to_s.gsub(/_(up|down)$/, '').to_sym
+          next if attr_name.length > 3 && self.up_down_reports.keys.include?(data_key)
+          # Else adds info to generic info
+          generic_info[attr_name] = content
+        end
+        generic_info[:name] = 'generic'
+        generic_info[:created_at] = created_at
+
+            res << {
+            index: "#{self.class.name.underscore.tr('/', '_')}_generic",
+            type: :info.to_s,
+            body: generic_info
+        }
+
+        res
+      end
+
+
+
+    end
+  end
+end

data/lib/internet_box_logger/parsers.rb

@@ -0,0 +1,12 @@
+require 'internet_box_logger/parsers/utils'
+require 'internet_box_logger/parsers/freebox_v5_parser'
+
+module InternetBoxLogger
+  module Parsers
+
+    def self.[]
+      constants.dup.keep_if {|c| const_get(c.to_s).is_a? Module and c.to_s =~ /\wParser$/ } .map {|m| const_get m}
+    end
+
+  end
+end

data/lib/internet_box_logger/version.rb

@@ -0,0 +1,3 @@
+module InternetBoxLogger
+  VERSION = "0.1.1"
+end

data/lib/internet_box_logger.rb

@@ -0,0 +1,17 @@
+require 'active_support/all'
+require 'elasticsearch'
+require 'easy_app_helper'
+
+require 'internet_box_logger/version'
+require 'internet_box_logger/elastic_search'
+require 'internet_box_logger/parsers'
+require 'internet_box_logger/generic_box'
+
+
+module InternetBoxLogger
+
+  def self.get_box(box_type=EasyAppHelper.config[:box_type])
+    InternetBoxLogger::GenericBox.new box_type
+  end
+
+end

data/lib/tasks/cron.rb

@@ -0,0 +1,28 @@
+module InternetBoxLogger
+  module Tasks
+    module Cron
+
+      def whenever_conf_file
+        "#{ibl_gem_path}/config/schedule.rb"
+      end
+
+      def cron_setup
+        EasyAppHelper.puts_and_logs "Using Whenever config file: '#{whenever_conf_file}' with interval #{EasyAppHelper.config[:cron_interval]}"
+        system "whenever -f '#{whenever_conf_file}' -i '#{whenever_conf_file}' -s interval='#{EasyAppHelper.config[:cron_interval]}'"
+        EasyAppHelper.puts_and_logs 'Crontab updated'
+      end
+
+      def cron_remove
+        EasyAppHelper.puts_and_logs "Using Whenever config file: '#{whenever_conf_file}'"
+        system "whenever -c '#{whenever_conf_file}'"
+        EasyAppHelper.puts_and_logs 'Crontab updated'
+      end
+
+      def cron_info
+        EasyAppHelper.puts_and_logs "Whenever config file used = #{whenever_conf_file}'"
+        EasyAppHelper.puts_and_logs "config.cron_interval = #{EasyAppHelper.config[:cron_interval]}"
+      end
+
+    end
+  end
+end

data/lib/tasks/elastic_search.rb

@@ -0,0 +1,67 @@
+module InternetBoxLogger
+  module Tasks
+    module ElasticSearch
+
+      ES_PID_FILE = '/tmp/es.pid'
+
+      def es_binary
+        es_bin_from_config = InternetBoxLogger::ElasticSearch::Server.local_path
+        es_bin = File.is_executable? es_bin_from_config
+        raise "Cannot find executable for ElasticSearch with name '#{es_bin_from_config}'. Try setting-up elastic_binary in application config." if es_bin.nil?
+        raise "You have not enough rights to run '#{es_bin_from_config}'." unless es_bin
+        es_bin
+      end
+
+      def already_running?
+        !es_pid.nil?
+      end
+
+      def es_pid
+        pid = `ps aux | grep 'elasticsearc[h]' | awk '{ print $2 }'`
+        return nil if pid.nil? || pid.empty?
+        pid.to_i
+      end
+
+
+      def create_pid_file(pid=nil)
+        if block_given?
+          File.open(ES_PID_FILE, 'w+') do |f|
+            yield f
+          end
+          pid = es_pid
+          raise "Invalid operation on pid file" if pid.nil? || pid < 1
+        else
+          raise "Specify a pid or a block !" if pid.nil?
+          raise "Invalid pid!" unless pid.is_a? Fixnum
+          File.open(ES_PID_FILE, 'w+') do |f|
+            f.puts pid
+          end
+        end
+        pid
+      end
+
+      def start_es_server
+        if already_running?
+          EasyAppHelper.puts_and_logs 'ElasticSearch already running... Aborting'
+          return nil
+        end
+        spawn("#{es_binary} -d")
+        EasyAppHelper.puts_and_logs 'ElasticSearch server started'
+        true
+      end
+
+      def stop_es_server
+        unless already_running?
+          EasyAppHelper.puts_and_logs 'ElasticSearch is not running... Nothing to stop'
+          return nil
+        end
+        pid = es_pid
+        raise 'Invalid operation on pid file' if pid.nil? || pid < 1
+        Process.kill('SIGTERM', pid)
+        EasyAppHelper.puts_and_logs 'ElasticSearch stopped'
+        true
+      end
+
+    end
+  end
+end

data/lib/tasks/file.rb

@@ -0,0 +1,26 @@
+class File
+
+  def self.is_executable?(filename)
+    real_name = nil
+    if exists?(filename)
+      real_name = filename
+    else
+      ENV['PATH'].split(':').each do |d|
+        f = join(d, filename)
+        if exists? f
+          real_name = f
+          break
+        end
+      end
+    end
+    return nil if real_name.nil? || real_name.empty?
+    executable_real?(real_name) ? real_name : false
+  end
+
+  def self.exists_in_path?(filename)
+    ENV['PATH'].split(':').collect do |d|
+      Dir.entries d if Dir.exists? d
+    end.flatten.include?(filename) ? filename : false
+  end
+
+end

data/lib/tasks/internet_box_logger_tasks.rake

@@ -0,0 +1,84 @@
+require 'rubygems'
+require 'bundler/setup'
+
+require File.expand_path '../internet_box_logger_tasks.rb', __FILE__
+
+
+namespace :internet_box_logger do
+
+  include InternetBoxLogger::Tasks
+
+  task :booted_environment do
+    require 'internet_box_logger'
+    EasyAppHelper.config.script_filename = "#{ibl_gem_path}/config/internet_box_logger.conf"
+    EasyAppHelper.config[:verbose] = true
+  end
+
+  # ElasticSearch tasks
+  namespace :es do
+
+    include InternetBoxLogger::Tasks::ElasticSearch
+
+    desc 'Starts your local ElasticSearch server'
+    task :start => :booted_environment  do
+      next unless start_es_server
+    end
+
+    desc 'Stops your local ElasticSearch server'
+    task :stop => :booted_environment do
+      next unless stop_es_server
+    end
+
+    desc 'Show your local ElasticSearch config'
+    task :info => :booted_environment do
+      EasyAppHelper.puts_and_logs "config.elastic_servers = #{InternetBoxLogger::ElasticSearch::Server[]}"
+      EasyAppHelper.puts_and_logs "config.elastic_binary = #{es_binary}"
+      EasyAppHelper.puts_and_logs "ElasticSearch server is currently #{%w{stopped running}[es_pid.nil? ? 0 : 1 ]}."
+    end
+  end
+
+
+  # Kibana tasks
+  namespace :kibana do
+
+    include InternetBoxLogger::Tasks::Kibana
+
+    desc 'Displays Kibana information'
+    task :info => :booted_environment do
+      kibana_info
+    end
+
+    desc 'Deploys box specific reports into Kibana dashboards directory'
+    task :deploy => :info do
+      deploy_reports
+    end
+
+    desc 'Launch a simple server to serve Kibana UI. You can specify the port as parameter'
+    task :serve, [:port]  => :info do |tsk, args|
+      serve_ui args[:port]
+    end
+
+  end
+
+  # Cron tasks
+  namespace :cron do
+
+    include InternetBoxLogger::Tasks::Cron
+
+    desc 'Setup cron to gather information every x minutes (configurable)'
+    task :setup => :booted_environment do
+      cron_setup
+    end
+
+    desc 'Removes cron task'
+    task :remove do
+      cron_remove
+    end
+
+    desc 'Show your Cron config'
+    task :info => :booted_environment do
+      cron_info
+    end
+  end
+
+end

data/lib/tasks/internet_box_logger_tasks.rb

@@ -0,0 +1,17 @@
+require 'tempfile'
+require File.expand_path '../file.rb', __FILE__
+require File.expand_path '../elastic_search.rb', __FILE__
+require File.expand_path '../cron.rb', __FILE__
+require File.expand_path '../kibana.rb', __FILE__
+
+
+module InternetBoxLogger
+  module Tasks
+
+    def ibl_gem_path
+      spec = Gem::Specification.find_by_name('internet_box_logger')
+      File.expand_path "../#{spec.name}", spec.spec_dir
+    end
+
+  end
+end