oxidized 0.1.1

data/lib/oxidized/node/stats.rb

@@ -0,0 +1,33 @@
+module Oxidized
+  class Node
+    class Stats
+      MAX_STAT = 10
+
+      # @param [Job] job job whose information add to stats
+      # @return [void]
+      def add job
+        stat = {
+          :start  => job.start,
+          :end    => job.end,
+          :time   => job.time,
+        }
+        @stats[job.status] ||= []
+        @stats[job.status].shift if @stats[job.status].size > MAX_STAT
+        @stats[job.status].push stat
+      end
+
+      # @param [Symbol] status stats for specific status
+      # @return [Hash,Array] Hash of stats for every status or Array of stats for specific status
+      def get status=nil
+        status ? @stats[status] : @stats
+      end
+
+      private
+
+      def initialize
+        @stats = {}
+      end
+
+    end
+  end
+end

data/lib/oxidized/nodes.rb

@@ -0,0 +1,143 @@
+module Oxidized
+ require 'oxidized/node'
+ require 'ipaddr'
+ class Oxidized::NotSupported < OxidizedError; end
+ class Oxidized::NodeNotFound < OxidizedError; end
+  class Nodes < Array
+    attr_accessor :source
+    alias :put :unshift
+    def load node_want=nil
+      with_lock do
+        new = []
+        node_want_ip = (IPAddr.new(node_want) rescue false) if node_want
+        @source = CFG.source.default
+        Oxidized.mgr.add_source @source
+        Oxidized.mgr.source[@source].new.load.each do |node|
+
+          # we want to load specific node(s), not all of them
+          if node_want
+            next unless node_want_ip == node[:ip] or node_want.match(node[:name])
+          end
+
+          begin
+            _node = Node.new node
+            new.push _node
+          rescue ModelNotFound => err
+            Log.error "node %s raised %s with message '%s'" % [node, err.class, err.message]
+          rescue Resolv::ResolvError => err
+            Log.error "node %s is not resolvable, raised %s with message '%s'" % [node, err.class, err.message]
+          end
+        end
+        size == 0 ? replace(new) : update_nodes(new)
+      end
+    end
+
+    def list
+      with_lock do
+        map { |e| e.serialize }
+      end
+    end
+
+    def show node
+      with_lock do
+        i = find_node_index node
+        self[i].serialize
+      end
+    end
+
+    def fetch node, group
+      with_lock do
+        i = find_node_index node
+        output = self[i].output.new
+        raise Oxidized::NotSupported unless output.respond_to? :fetch
+        output.fetch node, group
+      end
+    end
+
+    # @param node [String] name of the node moved into the head of array
+    def next node, opt={}
+      if waiting.find_node_index(node)
+        with_lock do
+          n = del node
+          n.user = opt['user']
+          n.msg  = opt['msg']
+          n.from = opt['from']
+          # set last job to nil so that the node is picked for immediate update
+          n.last = nil
+          put n
+        end
+      end
+    end
+    alias :top :next
+
+    # @return [String] node from the head of the array
+    def get
+      with_lock do
+        (self << shift).last
+      end
+    end
+
+    # @param node node whose index number in Nodes to find
+    # @return [Fixnum] index number of node in Nodes
+    def find_node_index node
+      find_index node or raise Oxidized::NodeNotFound, "unable to find '#{node}'"
+    end
+
+    private
+
+    def initialize opts={}
+      super()
+      node = opts.delete :node
+      @mutex= Mutex.new  # we compete for the nodes with webapi thread
+      if nodes = opts.delete(:nodes)
+        replace nodes
+      else
+        load node
+      end
+    end
+
+    def with_lock &block
+      @mutex.synchronize(&block)
+    end
+
+    def find_index node
+      index { |e| e.name == node }
+    end
+
+    # @param node node which is removed from nodes list
+    # @return [Node] deleted node
+    def del node
+      delete_at find_node_index(node)
+    end
+
+    # @return [Nodes] list of nodes running now
+    def running
+      Nodes.new :nodes => select { |node| node.running? }
+    end
+
+    # @return [Nodes] list of nodes waiting (not running)
+    def waiting
+      Nodes.new :nodes => select { |node| not node.running? }
+    end
+
+    # walks list of new nodes, if old node contains same name, adds last and
+    # stats information from old to new.
+    #
+    # @todo can we trust name to be unique identifier, what about when groups are used?
+    # @param [Array] nodes Array of nodes used to replace+update old
+    def update_nodes nodes
+      old = self.dup
+      replace(nodes)
+      each do |node|
+        begin
+          if i = old.find_node_index(node.name)
+            node.stats = old[i].stats
+            node.last  = old[i].last
+          end
+        rescue  Oxidized::NodeNotFound
+        end
+      end
+    end
+
+  end
+end

data/lib/oxidized/output/file.rb

@@ -0,0 +1,42 @@
+module Oxidized
+class OxidizedFile < Output
+  require 'fileutils'
+
+  def initialize
+    @cfg = CFG.output.file
+  end
+
+  def setup
+    if @cfg.empty?
+      CFGS.user.output.file.directory = File.join(Config::Root, 'configs')
+      CFGS.save :user
+      raise NoConfig, 'no output file config, edit ~/.config/oxidized/config'
+    end
+  end
+
+  def store node, data, opt={}
+    file = @cfg.directory
+    if opt[:group]
+      file = File.join File.dirname(file), opt[:group]
+    end
+    FileUtils.mkdir_p file
+    file = File.join file, node
+    open(file, 'w') { |fh| fh.write data }
+  end
+
+  def fetch node, group
+    cfg_dir = @cfg.directory
+    if group # group is explicitly defined by user
+      IO.readlines File.join(cfg_dir, group, node)
+    else
+      if File.exists? File.join(cfg_dir, node) # node configuration file is stored on base directory
+        IO.readlines File.join(cfg_dir, node)
+      else
+        path = Dir.glob File.join(cfg_dir, '**', node) # fetch node in all groups
+        open(path[0], 'r').readlines
+      end
+    end
+  end
+
+end
+end

data/lib/oxidized/output/git.rb

@@ -0,0 +1,78 @@
+module Oxidized
+class Git < Output
+  begin
+    require 'rugged'
+  rescue LoadError
+    raise OxidizedError, 'rugged not found: sudo gem install rugged'
+  end
+
+  def initialize
+    @cfg = CFG.output.git
+  end
+
+  def setup
+    if @cfg.empty?
+      CFGS.user.output.git.user  = 'Oxidized'
+      CFGS.user.output.git.email = 'o@example.com'
+      CFGS.user.output.git.repo  =  File.join(Config::Root, 'oxidized.git')
+      CFGS.save :user
+      raise NoConfig, 'no output git config, edit ~/.config/oxidized/config'
+    end
+  end
+
+  def store file, data, opt={}
+    msg   = opt[:msg]
+    user  = (opt[:user]  or @cfg.user)
+    email = (opt[:email] or @cfg.email)
+    repo  = @cfg.repo
+    if opt[:group]
+      repo = File.join File.dirname(repo), opt[:group] + '.git'
+    end
+    begin
+      repo = Rugged::Repository.new repo
+      update_repo repo, file, data, msg, user, email
+    rescue Rugged::OSError, Rugged::RepositoryError
+      Rugged::Repository.init_at repo, :bare
+      retry
+    end
+  end
+
+  def fetch node, group
+    begin
+      repo = @cfg.repo
+      if group
+        repo = File.join File.dirname(repo), group + '.git'
+      end
+      repo = Rugged::Repository.new repo
+      index = repo.index
+      index.read_tree repo.head.target.tree unless repo.empty?
+      repo.read(index.get(node)[:oid]).data
+    rescue
+      'node not found'
+    end
+  end
+
+  private
+
+  def update_repo repo, file, data, msg, user, email
+    oid = repo.write data, :blob
+    index = repo.index
+    index.read_tree repo.head.target.tree unless repo.empty?
+
+    tree_old = index.write_tree repo
+    index.add :path=>file, :oid=>oid, :mode=>0100644
+    tree_new = index.write_tree repo
+
+    if tree_old != tree_new
+      repo.config['user.name']  = user
+      repo.config['user.email'] = email
+      Rugged::Commit.create(repo,
+        :tree       => index.write_tree(repo),
+        :message    => msg,
+        :parents    => repo.empty? ? [] : [repo.head.target].compact,
+        :update_ref => 'HEAD',
+      )
+    end
+  end
+end
+end

data/lib/oxidized/output/output.rb

@@ -0,0 +1,5 @@
+module Oxidized
+  class Output
+    class NoConfig < OxidizedError; end
+  end
+end

data/lib/oxidized/source/csv.rb

@@ -0,0 +1,42 @@
+module Oxidized
+class CSV < Source
+  def initialize
+    @cfg = CFG.source.csv
+    super
+  end
+
+  def setup
+    if @cfg.empty?
+      CFGS.user.source.csv.file      = File.join(Config::Root, 'router.db')
+      CFGS.user.source.csv.delimiter = /:/
+      CFGS.user.source.csv.map.name  = 0
+      CFGS.user.source.csv.map.model = 1
+      CFGS.save :user
+      raise NoConfig, 'no source csv config, edit ~/.config/oxidized/config'
+    end
+  end
+
+  def load
+    nodes = []
+    open(@cfg.file).each_line do |line|
+      data  = line.chomp.split @cfg.delimiter
+      next if data.empty?
+      # map node parameters
+      keys = {}
+      @cfg.map.each do |key, position|
+        keys[key.to_sym] = data[position]
+      end
+      keys[:model] = map_model keys[:model] if keys.key? :model
+
+      # map node specific vars, empty value is considered as nil
+      vars = {}
+      @cfg.vars_map.each { |key, position| vars[key.to_sym] = data[position].to_s.empty? ? nil : data[position] }
+      keys[:vars] = vars unless vars.empty?
+
+      nodes << keys
+    end
+    nodes
+  end
+
+end
+end

data/lib/oxidized/source/source.rb

@@ -0,0 +1,11 @@
+module Oxidized
+  class Source
+    class NoConfig < OxidizedError; end
+    def initialize
+      @map = (CFG.model_map or {})
+    end
+    def map_model model
+      @map.has_key?(model) ? @map[model] : model
+    end
+  end
+end

data/lib/oxidized/source/sql.rb

@@ -0,0 +1,61 @@
+module Oxidized
+class SQL < Source
+  begin
+    require 'sequel'
+  rescue LoadError
+    raise OxidizedError, 'sequel not found: sudo gem install sequel'
+  end
+
+  def setup
+    if @cfg.empty?
+      CFGS.user.source.sql.adapter   = 'sqlite'
+      CFGS.user.source.sql.database  = File.join(Config::Root, 'sqlite.db')
+      CFGS.user.source.sql.table     = 'devices'
+      CFGS.user.source.sql.map.name  = 'name'
+      CFGS.user.source.sql.map.model = 'rancid'
+      CFGS.save :user
+      raise NoConfig, 'no source sql config, edit ~/.config/oxidized/config'
+    end
+  end
+
+  def load
+    nodes = []
+    db = connect
+    query = db[@cfg.table.to_sym]
+    query = query.with_sql(@cfg.query) if @cfg.query?
+    query.each do |node|
+      # map node parameters
+      keys = {}
+      @cfg.map.each { |key, sql_column| keys[key.to_sym] = node[sql_column.to_sym] }
+      keys[:model] = map_model keys[:model] if keys.key? :model
+
+      # map node specific vars
+      vars = {}
+      @cfg.vars_map.each { |key, sql_column| vars[key.to_sym] = node[sql_column.to_sym] }
+      keys[:vars] = vars unless vars.empty?
+
+      nodes << keys
+    end
+    db.disconnect
+    nodes
+  end
+
+  private
+
+  def initialize
+    super
+    @cfg = CFG.source.sql
+  end
+
+  def connect
+    Sequel.connect(:adapter  => @cfg.adapter,
+                   :host     => @cfg.host?,
+                   :user     => @cfg.user?,
+                   :password => @cfg.password?,
+                   :database => @cfg.database)
+  rescue Sequel::AdapterNotFound => error
+    raise OxidizedError, "SQL adapter gem not installed: " + error.message
+  end
+
+end
+end

data/lib/oxidized/string.rb

@@ -0,0 +1,13 @@
+module Oxidized
+  # Used in models, contains convenience methods
+  class String < String
+    # @return [Oxidized::String] copy of self with last line removed
+    def cut_tail
+      Oxy::String.new each_line.to_a[0..-2].join
+    end
+    # @return [Oxidized::String] copy of self with first line removed
+    def cut_head
+      Oxy::String.new each_line.to_a[1..-1].join
+    end
+  end
+end

data/lib/oxidized/worker.rb

@@ -0,0 +1,54 @@
+module Oxidized
+  require 'oxidized/job'
+  require 'oxidized/jobs'
+  class Worker
+    def initialize nodes
+      @nodes   = nodes
+      @jobs    = Jobs.new CFG.threads, CFG.interval, @nodes
+      Thread.abort_on_exception = true
+    end
+    def work
+      ended = []
+      @jobs.delete_if { |job| ended << job if not job.alive? }
+      ended.each      { |job| process job }
+      while @jobs.size < @jobs.want
+        Log.debug "Jobs #{@jobs.size}, Want: #{@jobs.want}"
+        # ask for next node in queue non destructive way
+        nextnode = @nodes.first
+        unless nextnode.last.nil?
+          break if nextnode.last.end + CFG.interval > Time.now.utc
+        end
+        # shift nodes and get the next node
+        node = @nodes.get
+        node.running? ? next : node.running = true
+        @jobs.push Job.new node
+      end
+    end
+    def process job
+      node = job.node
+      node.last = job
+      node.stats.add job
+      @jobs.duration job.time
+      node.running = false
+      if job.status == :success
+        msg = "update #{node.name}"
+        msg += " from #{node.from}" if node.from
+        msg += " with message '#{node.msg}'" if node.msg
+        node.output.new.store node.name, job.config,
+                              :msg => msg, :user => node.user, :group => node.group
+        node.reset
+      else
+        msg = "#{node.name} status #{job.status}"
+        if node.retry < CFG.retries
+          node.retry += 1
+          msg += ", retry attempt #{node.retry}"
+          @nodes.next node.name
+        else
+          msg += ", retries exhausted, giving up"
+          node.retry = 0
+        end
+        Log.warn msg
+      end
+    end
+  end
+end