observed 0.1.1 → 0.2.0.rc1

data/lib/observed/system.rb

@@ -7,6 +7,7 @@ module Observed
     def initialize(args={})
       @config = args[:config] if args[:config]
       @logger = args[:logger] if args[:logger]
+      @context = args[:context]
     end
 
     def config=(config)
@@ -17,93 +18,25 @@ module Observed
       @config
     end
 
-    def report(tag, time, data=nil)
-      if data.nil?
-        data = time
-        time = self.now
-      end
-      reporters.each do |reporter|
-        if reporter.match(tag)
-          reporter.report(tag, time, data)
-        end
-      end
-    end
-
-    def run(observation_name=nil)
-
+    def run(observation_name=nil, data=nil, options=nil)
+      options = { tag: (options && options[:tag]) || observation_name, time: now }.merge(options || {})
+      params = [data, options]
       if observation_name
-        observers_to_run = observers.reject { |o| o.tag != observation_name }
-        fail "No configuration found for observation name '#{observation_name}'" if observers_to_run.empty?
+        fail "No configuration found for observation name '#{observation_name}'" if @context.config_builder.group(observation_name).empty?
+        @context.config_builder.run_group(observation_name).send :now, *params
       else
-        observers_to_run = observers
-      end
-
-      observers_to_run.map do |input|
-        input.observe
+        observers_to_run = @context.config_builder.observers
+        fail "No configuration found for observation name '#{observation_name}'" if observers_to_run.empty?
+        observers_to_run.each do |o|
+          o.send :now, *params
+        end
       end
-
     end
 
     def now
       Time.now
     end
 
-    def logger
-      @logger ||= Logger.new(STDOUT)
-    end
-
-    private
-
-    def observers
-      config.observers
-    end
-
-    def reporters
-      config.reporters
-    end
-
   end
 
-  class YAML
-    def observers
-      config.observers
-      @observers ||= begin
-
-        observer_configs = config.observers
-
-        observers = {}
-
-        observer_configs.each do |tag, input_config|
-          plugin_name = input_config[:plugin] || fail(RuntimeError, %Q|Missing plugin name for the tag "#{tag}" in "#{input_config}" in "#{config}".|)
-          plugin = observer_plugins[plugin_name] || fail(RuntimeError, %Q|The plugin named "#{plugin_name}" is not found in plugins list "#{observer_plugins}".|)
-          updated_config = input_config.merge(tag: tag)
-          observer = plugin.new(updated_config)
-          observer.configure(system: self, logger: logger)
-          observers[tag] = observer
-        end
-
-        observers
-      end
-    end
-
-    def reporters
-      @reporters ||= begin
-
-        reporter_configs = config.reporters
-
-        reporters = {}
-
-        reporter_configs.each do |tag_pattern, output_config|
-          plugin_name = output_config[:plugin] || fail(RuntimeError, %Q|Missing plugin name for the output for "#{tag_pattern}" in "#{output_config}" in #{config}.|)
-          plugin = reporter_plugins[plugin_name] || fail(RuntimeError, %Q|The plugin named "#{plugin_name}" is not found in plugins list "#{reporter_plugins}".|)
-          updated_config = output_config.merge(tag_pattern: Regexp.new(tag_pattern))
-          reporter = plugin.new(updated_config)
-          reporter.configure(system: self, logger: logger)
-          reporters[tag_pattern] = reporter
-        end
-
-        reporters
-      end
-    end
-  end
 end

data/lib/observed/translator.rb

@@ -0,0 +1,22 @@
+require 'observed/configurable'
+require 'observed/logging'
+require 'observed/pluggable'
+
+module Observed
+  class Translator
+
+    include Observed::Configurable
+    include Observed::Logging
+    include Observed::Pluggable
+
+    attribute :tag_pattern
+
+    def match(tag)
+      tag_pattern.match(tag)
+    end
+
+    def translate(tag, time, data)
+      fail RuntimeError, 'Not implemented method: Observed#translate'
+    end
+  end
+end

data/lib/observed/version.rb

@@ -1,3 +1,3 @@
 module Observed
-  VERSION = '0.1.1'
+  VERSION = '0.2.0.rc1'
 end

data/lib/observed.rb

@@ -1,7 +1,5 @@
 require 'observed/version'
-require 'observed/config_builder'
-require 'observed/system'
-require 'observed/config_dsl'
+require 'observed/context'
 require 'observed/builtin_plugins'
 require 'forwardable'
 
@@ -28,24 +26,24 @@ module Observed
   class Singleton
     extend Forwardable
 
-    def_delegators :@observed, :require_relative, :observe, :report, :write, :read, :config, :load!, :working_directory
+    def_delegators :@observed, :require_relative, :observe, :translate, :report, :write, :read, :emit, :receive, :group, :run_group, :config, :load!, :working_directory
 
     # Call this method before you are going to build 2nd or later Observed configuration using this module.
     # Refrain that `Observed` object is a builder for Observed configuration and it has global state.
     # We have to reset its state via this `init!` method before building next configurations after the first one.
     def init!
-      @sys = Observed::System.new
-      config_builder = Observed::ConfigBuilder.new(system: @sys)
-      @observed = Observed::ConfigDSL.new(builder: config_builder)
+      @context = Observed::Context.new
+      @observed = @context.config_dsl
     end
 
-    def run(tag=nil)
-      @sys.config = self.config
-      @sys.run(tag)
+    def run(tag=nil, data=nil, options=nil)
+      sys = @context.system
+      sys.config = @observed.config
+      sys.send :run, *[tag, data, options].take_while { |a| a != nil }
     end
 
     def configure(*args)
-      @observed.send :configure, *args
+      @context.configure *args
     end
   end
 
@@ -69,7 +67,7 @@ module Observed
 
   extend Forwardable
 
-  def_delegators :@@singleton, :run, :init!, :configure, :require_relative, :observe, :report, :write, :read, :config,
+  def_delegators :@@singleton, :run, :init!, :configure, :require_relative, :observe, :translate, :report, :write, :read, :emit, :receive, :group, :run_group, :config,
                  :load!, :working_directory
 
   extend self

data/omnibus-observed/.gitignore

@@ -0,0 +1,9 @@
+*.gem
+.bundle
+vendor/bundle
+pkg/*
+.vagrant
+bin/*
+files/**/cache/
+vendor/cookbooks
+*~

data/omnibus-observed/Berksfile

@@ -0,0 +1,3 @@
+site :opscode
+
+cookbook "omnibus"

data/omnibus-observed/Berksfile.lock

@@ -0,0 +1,52 @@
+{
+  "sources": {
+    "omnibus": {
+      "locked_version": "1.2.0"
+    },
+    "apt": {
+      "locked_version": "1.9.2"
+    },
+    "build-essential": {
+      "locked_version": "1.4.2"
+    },
+    "git": {
+      "locked_version": "2.3.0"
+    },
+    "dmg": {
+      "locked_version": "2.0.8"
+    },
+    "yum": {
+      "locked_version": "2.4.2"
+    },
+    "windows": {
+      "locked_version": "1.11.0"
+    },
+    "chef_handler": {
+      "locked_version": "1.1.4"
+    },
+    "runit": {
+      "locked_version": "1.4.0"
+    },
+    "homebrew": {
+      "locked_version": "1.3.2"
+    },
+    "pkgin": {
+      "locked_version": "0.4.0"
+    },
+    "pkgutil": {
+      "locked_version": "0.0.3"
+    },
+    "rbenv": {
+      "locked_version": "1.6.5"
+    },
+    "ohai": {
+      "locked_version": "1.1.12"
+    },
+    "wix": {
+      "locked_version": "1.1.0"
+    },
+    "7-zip": {
+      "locked_version": "1.0.2"
+    }
+  }
+}

data/omnibus-observed/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+gem 'omnibus', '~> 1.2.0'
+gem 'omnibus-software', :git => 'git://github.com/opscode/omnibus-software.git', :branch => 'master'

data/omnibus-observed/README.md

@@ -0,0 +1,102 @@
+# observed Omnibus project
+
+This project creates full-stack platform-specific packages for
+`observed`!
+
+## Installation
+
+We'll assume you have Ruby 1.9+ and Bundler installed. First ensure all
+required gems are installed and ready to use:
+
+```shell
+$ bundle install --binstubs
+```
+
+## Usage
+
+### Build
+
+You create a platform-specific package using the `build project` command:
+
+```shell
+$ bin/omnibus build project observed
+```
+
+The platform/architecture type of the package created will match the platform
+where the `build project` command is invoked. So running this command on say a
+MacBook Pro will generate a Mac OS X specific package. After the build
+completes packages will be available in `pkg/`.
+
+### Clean
+
+You can clean up all temporary files generated during the build process with
+the `clean` command:
+
+```shell
+$ bin/omnibus clean
+```
+
+Adding the `--purge` purge option removes __ALL__ files generated during the
+build including the project install directory (`/opt/observed`) and
+the package cache directory (`/var/cache/omnibus/pkg`):
+
+```shell
+$ bin/omnibus clean --purge
+```
+
+### Help
+
+Full help for the Omnibus command line interface can be accessed with the
+`help` command:
+
+```shell
+$ bin/omnibus help
+```
+
+## Vagrant-based Virtualized Build Lab
+
+Every Omnibus project ships will a project-specific
+[Berksfile](http://berkshelf.com/) and [Vagrantfile](http://www.vagrantup.com/)
+that will allow you to build your projects on the following platforms:
+
+* CentOS 5 64-bit
+* CentOS 6 64-bit
+* Ubuntu 10.04 64-bit
+* Ubuntu 11.04 64-bit
+* Ubuntu 12.04 64-bit
+
+Please note this build-lab is only meant to get you up and running quickly;
+there's nothing inherent in Omnibus that restricts you to just building CentOS
+or Ubuntu packages. See the Vagrantfile to add new platforms to your build lab.
+
+The only requirements for standing up this virtualized build lab are:
+
+* VirtualBox - native packages exist for most platforms and can be downloaded
+from the [VirtualBox downloads page](https://www.virtualbox.org/wiki/Downloads).
+* Vagrant 1.2.1+ - native packages exist for most platforms and can be downloaded
+from the [Vagrant downloads page](http://downloads.vagrantup.com/).
+
+The [vagrant-berkshelf](https://github.com/RiotGames/vagrant-berkshelf) and
+[vagrant-omnibus](https://github.com/schisamo/vagrant-omnibus) Vagrant plugins
+are also required and can be installed easily with the following commands:
+
+```shell
+$ vagrant plugin install vagrant-berkshelf
+$ vagrant plugin install vagrant-omnibus
+```
+
+Once the pre-requisites are installed you can build your package across all
+platforms with the following command:
+
+```shell
+$ vagrant up
+```
+
+If you would like to build a package for a single platform the command looks like this:
+
+```shell
+$ vagrant up PLATFORM
+```
+
+The complete list of valid platform names can be viewed with the
+`vagrant status` command.

data/omnibus-observed/Vagrantfile

@@ -0,0 +1,93 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+require "vagrant"
+
+if Vagrant::VERSION < "1.2.1"
+  raise "The Omnibus Build Lab is only compatible with Vagrant 1.2.1+"
+end
+
+host_project_path = File.expand_path("..", __FILE__)
+guest_project_path = "/home/vagrant/#{File.basename(host_project_path)}"
+project_name = "observed"
+
+Vagrant.configure("2") do |config|
+
+  config.vm.hostname = "#{project_name}-omnibus-build-lab"
+
+  config.vm.define 'ubuntu-10.04' do |c|
+    c.berkshelf.berksfile_path = "./Berksfile"
+    c.vm.box = "opscode-ubuntu-10.04"
+    c.vm.box_url = "http://opscode-vm.s3.amazonaws.com/vagrant/opscode_ubuntu-10.04_chef-11.2.0.box"
+  end
+
+  config.vm.define 'ubuntu-11.04' do |c|
+    c.berkshelf.berksfile_path = "./Berksfile"
+    c.vm.box = "opscode-ubuntu-11.04"
+    c.vm.box_url = "http://opscode-vm.s3.amazonaws.com/vagrant/boxes/opscode-ubuntu-11.04.box"
+  end
+
+  config.vm.define 'ubuntu-12.04' do |c|
+    c.berkshelf.berksfile_path = "./Berksfile"
+    c.vm.box = "canonical-ubuntu-12.04"
+    c.vm.box_url = "http://cloud-images.ubuntu.com/vagrant/precise/current/precise-server-cloudimg-amd64-vagrant-disk1.box"
+  end
+
+  config.vm.define 'centos-5' do |c|
+    c.berkshelf.berksfile_path = "./Berksfile"
+    c.vm.box = "opscode-centos-5.8"
+    c.vm.box_url = "http://opscode-vm.s3.amazonaws.com/vagrant/opscode_centos-5.8_chef-11.2.0.box"
+  end
+
+  config.vm.define 'centos-6' do |c|
+    c.berkshelf.berksfile_path = "./Berksfile"
+    c.vm.box = "opscode-centos-6.3"
+    c.vm.box_url = "http://opscode-vm.s3.amazonaws.com/vagrant/opscode_centos-6.3_chef-11.2.0.box"
+  end
+
+  config.vm.provider :virtualbox do |vb|
+    # Give enough horsepower to build without taking all day.
+    vb.customize [
+      "modifyvm", :id,
+      "--memory", "1536",
+      "--cpus", "2"
+    ]
+  end
+
+  # Ensure a recent version of the Chef Omnibus packages are installed
+  config.omnibus.chef_version = :latest
+
+  # Enable the berkshelf-vagrant plugin
+  config.berkshelf.enabled = true
+  # The path to the Berksfile to use with Vagrant Berkshelf
+  config.berkshelf.berksfile_path = "./Berksfile"
+
+  config.ssh.forward_agent = true
+
+  host_project_path = File.expand_path("..", __FILE__)
+  guest_project_path = "/home/vagrant/#{File.basename(host_project_path)}"
+
+  config.vm.synced_folder host_project_path, guest_project_path
+
+  # prepare VM to be an Omnibus builder
+  config.vm.provision :chef_solo do |chef|
+    chef.json = {
+      "omnibus" => {
+        "build_user" => "vagrant",
+        "build_dir" => guest_project_path,
+        "install_dir" => "/opt/#{project_name}"
+      }
+    }
+
+    chef.run_list = [
+      "recipe[omnibus::default]"
+    ]
+  end
+
+  config.vm.provision :shell, :inline => <<-OMNIBUS_BUILD
+    export PATH=/usr/local/bin:$PATH
+    cd #{guest_project_path}
+    su vagrant -c "bundle install --binstubs"
+    su vagrant -c "bin/omnibus build project #{project_name}"
+  OMNIBUS_BUILD
+end

data/omnibus-observed/config/projects/observed.rb

@@ -0,0 +1,20 @@
+
+name "observed"
+maintainer "KUOKA Yusuke"
+homepage "https://github.com/gree/observed"
+
+replaces        "observed"
+install_path    "/opt/observed"
+build_version   Omnibus::BuildVersion.new.semver
+build_iteration 1
+
+# creates required build directories
+dependency "preparation"
+
+dependency "observed"
+
+# version manifest file
+dependency "version-manifest"
+
+exclude "\.git*"
+exclude "bundler\/git"

data/omnibus-observed/config/software/observed.rb

@@ -0,0 +1,19 @@
+name "observed"
+#version "0.1.1"
+version "master"
+
+dependency "ruby"
+dependency "rubygems"
+#dependency "bundler"
+dependency "rsync"
+
+source :git => "https://github.com/gree/observed.git"
+
+relative_path "observed"
+
+build do
+  gem "install bundler --no-rdoc --no-ri -v 1.3.0"
+  bundle "install --path=#{install_dir}/embedded/service/gem"
+  command "mkdir -p #{install_dir}/embedded/service/observed"
+  command "#{install_dir}/embedded/bin/rsync -a --delete --exclude=.git/*** --exclude=.gitignore ./ #{install_dir}/embedded/service/observed/"
+end

data/omnibus-observed/package-scripts/observed/makeselfinst

@@ -0,0 +1,27 @@
+#!/bin/bash
+#
+# Install a full observed
+#
+
+PROGNAME=`basename $0`
+INSTALLER_DIR=`dirname $0`
+DEST_DIR=/opt/observed
+CONFIG_DIR=/etc/observed
+USAGE="usage: $0"
+
+error_exit()
+{
+  echo "${PROGNAME}: ${1:-"Unknown Error"}" 1>&2
+  exit 1
+}
+
+# move the actual files into place
+rm -rf $DEST_DIR/* || error_exit "Cannot remove contents of $DEST_DIR"
+mkdir -p $DEST_DIR || error_exit "Cannot create $DEST_DIR"
+cp -R $INSTALLER_DIR $DEST_DIR || error_exit "Cannot install to $DEST_DIR"
+rm -f $DEST_DIR/$PROGNAME
+
+# You may want to symlink your packages bin files into /usr/bin
+# ln -sf $DEST_DIR/bin/observed /usr/bin || error_exit "Cannot link observed to /usr/bin"
+
+exit 0

data/omnibus-observed/package-scripts/observed/postinst

@@ -0,0 +1,17 @@
+#!/bin/bash
+#
+# Perform necessary observed setup steps
+# after package is installed.
+#
+
+PROGNAME=$(basename $0)
+
+function error_exit
+{
+  echo "${PROGNAME}: ${1:-"Unknown Error"}" 1>&2
+  exit 1
+}
+
+echo "Thank you for installing observed!"
+
+exit 0

data/omnibus-observed/package-scripts/observed/postrm

@@ -0,0 +1,9 @@
+#!/bin/bash
+#
+# Perform necessary observed removal steps
+# after package is uninstalled.
+#
+
+echo "observed has been uninstalled!"
+
+exit 0

data/plugins/observed-fluentd/lib/observed/fluentd/version.rb

@@ -1,5 +1,5 @@
 module Observed
   class Fluentd
-    VERSION = "0.1.0"
+    VERSION = "0.2.0.rc1"
   end
 end

data/plugins/observed-gauge/README.md

@@ -16,6 +16,11 @@ Or install it yourself as:
 
     $ gem install observed-gauge
 
+You might also need to install rrdtool:
+
+    (CentOS or Ubuntu)
+    $ sudo apt-get install librrd-dev
+
 ## Usage
 
 TODO: Write usage instructions here

data/plugins/observed-gauge/lib/observed/gauge/version.rb

@@ -1,5 +1,5 @@
 module Observed
   module Gauge
-    VERSION = "0.1.0"
+    VERSION = "0.2.0.rc1"
   end
 end

data/plugins/observed-gauge/lib/observed/gauge.rb

@@ -1,4 +1,5 @@
-require 'observed/reporter'
+require 'observed/logging'
+require 'observed/translator'
 require 'observed/reporter/regexp_matching'
 require 'observed/gauge/version'
 require 'logger'
@@ -6,39 +7,40 @@ require 'rrd'
 
 module Observed
   module Plugins
-    class Gauge < Observed::Reporter
+    class Gauge < Observed::Translator
 
       plugin_name 'gauge'
 
+      include Observed::Logging
       include Observed::Reporter::RegexpMatching
 
-      attribute :tag
       attribute :key_path
       attribute :coerce, default: ->(data){ data }
       attribute :rrd
       attribute :step
       attribute :period
 
-      def report(tag, time, data)
+      def translate(data, options)
+        time = options[:time] || Time.now
         rewrote = update_value_for_key_path(data, key_path) do |v|
           sample = coerce.call(v)
           average = get_cdp_updated_with(time, sample)
           average
         end
         unless fetch_value_for_key_path(rewrote, key_path).nan?
-          system.report(self.tag, rewrote)
+          rewrote
         end
       end
 
       def prepare_rrd(args)
         start = args[:start]
-        logger.debug "Creating a rrd file named '#{args[:rrd]}' with options {:start => #{start}}"
+        log_debug "Creating a rrd file named '#{args[:rrd]}' with options {:start => #{start}}"
         result = RRD::Builder.new(args[:rrd], start: start, step: step.seconds).tap do |builder|
           builder.datasource data_source, :type => :gauge, :heartbeat => period.seconds, :min => 0, :max => :unlimited
           builder.archive :average, :every => period.seconds, :during => period.seconds
           builder.save
         end
-        logger.debug "Builder#save returned: #{result.inspect}"
+        log_debug "Builder#save returned: #{result.inspect}"
       end
 
       private
@@ -101,17 +103,13 @@ module Observed
           prepare_rrd(rrd: rrd_path, start: t)
         end
 
-        logger.debug "Updating the data source '#{data_source}' with the value #{value} with timestamp #{t}"
+        log_debug "Updating the data source '#{data_source}' with the value #{value} with timestamp #{t}"
         rrd.update t, value
 
-        logger.debug rrd.fetch!(:average)[-2..-1]
+        log_debug rrd.fetch!(:average)[-2..-1]
 
         rrd.fetch(:average)[-2..-1].first.last
       end
-
-      def logger
-        @logger ||= Logger.new(STDOUT)
-      end
     end
   end
 end

data/plugins/observed-gauge/observed-gauge.gemspec

@@ -18,7 +18,7 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ['lib']
 
-  spec.add_dependency 'observed', '~> 0.1.0'
+  spec.add_dependency 'observed', '~> 0.2.0.rc1'
   spec.add_dependency 'rrd-ffi', '~> 0.2.14'
 
   spec.add_development_dependency 'bundler', '~> 1.3'