vines-services 0.1.0

data/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2010-2011 Negative Code
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README

@@ -0,0 +1,40 @@
+== Welcome to Vines Services
+
+Vines Services are dynamically updated groups of systems based on criteria like
+hostname, installed software, operating system, etc. Send a command to the
+service and it runs on every system in the group. Services, files and permissions
+are managed via the bundled web application.
+
+Services are defined using VinesQL, a simple query language that finds systems
+based on their machine attributes. A "Mac OS X Lion" service might be
+defined as: platform is 'mac_os_x' and platform_version starts with '10.7'.
+As machines update to Lion, they will join this service automatically and may
+be managed as one group.
+
+== Usage
+
+1. gem install vines-services
+2. vines-services init wonderland.lit
+3. cd wonderland.lit && vines-services start
+
+== Dependencies
+
+* bcrypt-ruby >= 3.0.0
+* blather >= 0.5.4
+* citrus >= 2.4.0
+* eventmachine >= 0.12.10
+* nokogiri >= 1.4.4
+* sqlite3 >= 1.3.4
+* ruby >= 1.9.2
+
+== Ubuntu setup
+$ sudo apt-get install build-essential ruby1.9.1 ruby1.9.1-dev libxml2-dev libxslt-dev
+
+== Contact
+
+* David Graham  <david@negativecode.com>
+* Chris Johnson <chris@negativecode.com>
+
+== License
+
+Vines Services is released under the MIT license. Check the LICENSE file for details.

data/Rakefile

@@ -0,0 +1,130 @@
+require 'rake'
+require 'rake/clean'
+require 'rake/testtask'
+require 'rubygems/package_task'
+require 'nokogiri'
+require_relative 'lib/vines/services/version'
+
+spec = Gem::Specification.new do |s|
+  s.name    = "vines-services"
+  s.version = Vines::Services::VERSION
+  s.date    = Time.now.strftime("%Y-%m-%d")
+
+  s.summary     = "An XMPP component that broadcasts shell commands to many agents."
+  s.description = "Vines Services are dynamically updated groups of systems based
+on criteria like hostname, installed software, operating system, etc. Send a
+command to the service and it runs on every system in the group. Services, files
+and permissions are managed via the bundled web application."
+
+  s.authors      = ["David Graham", "Chris Johnson"]
+  s.email        = %w[david@negativecode.com chris@negativecode.com]
+  s.homepage     = "http://www.getvines.com"
+
+  s.files        = FileList['[A-Z]*', '{bin,lib,conf,web}/**/*']
+  s.test_files   = FileList["test/**/*"]
+  s.executables  = %w[vines-services]
+  s.require_path = "lib"
+
+  s.add_dependency "bcrypt-ruby", "~> 3.0.0"
+  s.add_dependency "blather", "~> 0.5.4"
+  s.add_dependency "citrus", "~> 2.4.0"
+  s.add_dependency "couchrest_model", "~> 1.1.2"
+  s.add_dependency "em-http-request", "~> 0.3.0"
+  s.add_dependency "sqlite3", "~> 1.3.4"
+  s.add_dependency "vines", "~> 0.3"
+
+  s.add_development_dependency "minitest"
+  s.add_development_dependency "rake"
+
+  s.required_ruby_version = '>= 1.9.2'
+end
+
+Gem::PackageTask.new(spec) do |pkg|
+  pkg.need_tar = true
+end
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'test'
+  test.pattern = 'test/**/*_test.rb'
+  test.warning = false
+end
+
+# Return an array of local, non-library, js includes.
+def scripts(doc)
+  scripts = []
+  doc.css('script').each do |node|
+    file = node['src'].split('/').last()
+    if node['src'].start_with?('javascripts/')
+      scripts << file
+    end
+  end
+  scripts
+end
+
+# Replace script tags with combined and minimized files.
+def rewrite_js(doc)
+  doc.css('script').each {|node| node.remove }
+  doc.css('head').each do |node|
+    %w[/lib/javascripts/base.js javascripts/app.js].each do |src|
+      script = doc.create_element('script',
+        'type' => 'text/javascript',
+        'src' => src)
+      node.add_child(script)
+      node.add_child(doc.create_text_node("\n"))
+    end
+  end
+end
+
+def stylesheets(doc)
+  sheets = []
+  doc.css('link[rel="stylesheet"]').each do |node|
+    file = node['href'].split('/').last()
+    if node['href'].start_with?('stylesheets/')
+      sheets << file
+    end
+  end
+  sheets
+end
+
+def rewrite_css(doc)
+  doc.css('link[rel="stylesheet"]').each {|node| node.remove }
+  doc.css('head').each do |node|
+    %w[/lib/stylesheets/base.css /lib/stylesheets/login.css stylesheets/app.css].each do |file|
+      link = doc.create_element('link',
+        'rel' => 'stylesheet',
+        'href' => file)
+      node.add_child(link)
+      node.add_child(doc.create_text_node("\n"))
+    end
+  end
+end
+
+task :compile do
+  index = 'web/index.html'
+  doc = Nokogiri::HTML(File.read(index))
+  scripts, sheets = scripts(doc), stylesheets(doc)
+
+  rewrite_js(doc)
+  rewrite_css(doc)
+
+  # save index.html before rewriting
+  FileUtils.cp(index, '/tmp/index.html')
+  File.open(index, 'w') {|f| f.write(doc.to_xml(:indent => 2)) }
+
+  js_files = scripts.map {|f| "web/javascripts/#{f}"}.join(' ')
+  css_files = sheets.map {|f| "web/stylesheets/#{f}"}.join(' ')
+
+  sh %{coffee -c -b -o web/javascripts web/coffeescripts/*.coffee}
+  sh %{cat #{js_files} | uglifyjs -nc > web/javascripts/app.js}
+
+  sh %{cat #{css_files} > web/stylesheets/app.css}
+end
+
+task :cleanup do
+  # move index.html back into place after gem packaging
+  FileUtils.cp('/tmp/index.html', 'web/index.html')
+  File.delete('/tmp/index.html')
+end
+
+
+task :default => [:clobber, :test, :compile, :gem, :cleanup]

data/bin/vines-services

@@ -0,0 +1,95 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+
+require 'optparse'
+require 'vines/services'
+
+COMMANDS = %w[start stop restart init views]
+
+def parse(args)
+  options = {}
+  parser = OptionParser.new do |opts|
+    opts.banner = "Usage: vines-services [options] #{COMMANDS.join('|')}"
+
+    opts.separator ""
+    opts.separator "Daemon options:"
+
+    opts.on('-d', '--daemonize', 'Run daemonized in the background') do |daemonize|
+      options[:daemonize] = daemonize
+    end
+
+    options[:log] = 'log/vines-services.log'
+    opts.on('-l', '--log FILE', 'File to redirect output (default: log/vines-services.log)') do |log|
+      options[:log] = log
+    end
+
+    options[:pid] = 'pid/vines-services.pid'
+    opts.on('-P', '--pid FILE', 'File to store PID (default: pid/vines-services.pid)') do |pid|
+      options[:pid] = pid
+    end
+
+    opts.separator ""
+    opts.separator "Common options:"
+
+    opts.on('-h', '--help', 'Show this message') do |help|
+      options[:help] = help
+    end
+
+    opts.on('-v', '--version', 'Show version') do |version|
+      options[:version] = version
+    end
+  end
+
+  begin
+    parser.parse!(args)
+  rescue
+    puts parser
+    exit(1)
+  end
+
+  if options[:version]
+    puts Vines::Services::VERSION
+    exit
+  end
+
+  if options[:help]
+    puts parser
+    exit
+  end
+
+  command = args.shift
+  unless COMMANDS.include?(command)
+    puts parser
+    exit(1)
+  end
+
+  options.tap do |opts|
+    opts[:args]    = args
+    opts[:command] = command
+    opts[:config]  = File.expand_path('conf/config.rb')
+    opts[:pid]     = File.expand_path(opts[:pid])
+    opts[:log]     = File.expand_path(opts[:log])
+  end
+end
+
+def check_config(opts)
+  return if %w[init].include?(opts[:command])
+  unless File.exists?(opts[:config])
+    puts "No config file found at #{opts[:config]}"
+    exit(1)
+  end
+end
+
+VINES_ROOT = Dir.pwd
+opts = parse(ARGV)
+check_config(opts)
+command = Vines::Services::Command.const_get(opts[:command].capitalize).new
+begin
+  command.run(opts)
+rescue SystemExit
+  # do nothing
+rescue Exception => e
+  puts e.message
+  puts e.backtrace
+  exit(1)
+end

data/conf/config.rb

@@ -0,0 +1,25 @@
+# encoding: UTF-8
+
+# This is the Vines Services configuration file. Restart the service with
+# 'vines-services restart' after updating this file.
+
+Vines::Services::Config.configure do
+  # Set the logging level to debug, info, warn, error, or fatal. The debug
+  # level logs all XML sent and received by the server.
+  log :info
+
+  host 'vines.wonderland.lit' do
+    upstream 'localhost', 5347, 'secr3t'
+    uploads 'data/upload'
+
+    storage 'couchdb' do
+      host 'localhost'
+      port 5984
+      database 'xmpp'
+      tls false
+      username ''
+      password ''
+      index_dir 'data/index'
+    end
+  end
+end

data/lib/vines/services/command/init.rb

@@ -0,0 +1,209 @@
+# encoding: UTF-8
+
+module Vines
+  module Services
+    module Command
+      class Init
+        def run(opts)
+          raise 'vines-services init [domain]' unless opts[:args].size == 1
+          @domain = opts[:args].first.dup
+          dir = File.expand_path(@domain)
+          @component_password = Kit.generate_password
+          create_directories
+          initialize_server
+          initialize_component
+          initialize_agent
+          puts "Initialized service, agent, and server directories: #{@domain}"
+        end
+
+        private
+
+        def initialize_server
+          Dir.chdir(File.join(@domain, "server"))
+          `vines init #{@domain}`
+          configure_database
+          FileUtils.mv Dir.glob("#{@domain}/*"), "./"
+          FileUtils.remove_dir(@domain)
+          `vines start -d`
+          FileUtils.cp_r(File.expand_path("../../../../../web/index.html", __FILE__), File.join("web", "index.html"))
+          FileUtils.cp_r(File.expand_path("../../../../../web/javascripts", __FILE__), File.join("web", "javascripts"))
+          FileUtils.cp_r(File.expand_path("../../../../../web/stylesheets", __FILE__), File.join("web", "stylesheets"))
+          FileUtils.cp_r(File.expand_path("../../../../../web/images", __FILE__), File.join("web", "images"))
+          FileUtils.cp_r(File.expand_path("../../../../../web/coffeescripts", __FILE__), File.join("web", "coffeescripts"))
+        end
+
+        def initialize_agent
+          Dir.chdir("../agent/")
+          `vines-agent init #{@domain}`
+          FileUtils.cp("../server/conf/certs/#{@domain}.crt", File.join("#{@domain}", "conf", "certs"))
+          agent_password = Kit.generate_password
+          jid = Vines::JID.new(fqdn, @domain).bare
+          id = "user:#{jid}"
+          Fiber.new do
+            user = Vines::Services::CouchModels::User.new(id: id)
+            user.name = jid
+            user.system = true
+            user.password = agent_password
+            if user.valid?
+              user.save
+              log.debug("Saving user: #{user}")
+            end
+          end.resume
+          update_agent_config(File.join("#{@domain}", "conf", "config.rb"), agent_password)
+          FileUtils.mv Dir.glob("#{@domain}/*"), "./"
+          FileUtils.remove_dir(@domain)
+          `vines-agent start -d`
+        end
+
+        # Coordinate all the configuration required for the service
+        def initialize_component
+          Dir.chdir("../services/")
+          %w[conf].each do |sub|
+            FileUtils.cp_r(File.expand_path("../../../../../#{sub}", __FILE__), File.join("."))
+          end
+          log, pid = %w[log pid data data/index].map do |sub|
+            File.join(sub).tap {|subdir| Dir.mkdir(subdir) }
+          end
+          update_config(File.join("conf", "config.rb"))
+          `vines-services start -d`
+        end
+
+        #create all the directories for the server, agent, and service
+        def create_directories
+          Dir.mkdir(@domain)
+          %w[server services agent].each do |sub|
+            Dir.mkdir(File.join(@domain, sub))
+          end
+        end
+
+        def create_db(host, port, db)
+          url = "http://#{host}:#{port}/#{db}"
+          begin
+            RestClient.head(url)
+          rescue RestClient::ResourceNotFound
+            RestClient.put(url, nil)
+          end
+          true
+        rescue
+          false
+        end
+
+        # This will attempt to create a new couch database. If one exists
+        # by the name generated, it will not be deleted. This will loop until
+        # it gets a valid couch db address and port
+        def configure_database
+          server, port = "127.0.0.1", 5984
+          db_name = @domain.gsub(".", "_").downcase()
+          until create_db(server, port, db_name)
+            puts "Unable to connect."
+            $stdout.write('CouchDB Server: ')
+            server = $stdin.gets.chomp
+            $stdout.write('CouchDB Port: ')
+            port = $stdin.gets.chomp
+          end
+          @couch_server = server
+          @couch_port = port
+          @couch_db = db_name
+          update_config(File.join(@domain, "conf", "config.rb"))
+          @storage = Vines::Services::Storage::CouchDB.new do
+            host server
+            port port
+            database db_name
+            index_dir '.'
+          end
+          create_user
+        end
+
+        #Create a new vines user object in the couch database
+        def create_user
+          jid, password = nil, nil
+          until jid
+            $stdout.write('JID: ')
+            if node = $stdin.gets.chomp.split('@').first
+              jid = Vines::JID.new(node, @domain) rescue nil
+            end
+          end
+
+          until password
+            $stdout.write('Password: ')
+            `stty -echo`
+            password = $stdin.gets.chomp
+            password = nil if password.empty?
+            `stty echo`
+          end
+          puts "\nCreated #{jid}"
+
+          id = "user:#{jid}"
+          Fiber.new do
+            user = Vines::Services::CouchModels::User.new(id: id)
+            user.name = jid
+            user.password = password
+            if user.valid?
+              user.save
+              log.debug("\nSaving user: #{user}")
+            end
+          end.resume
+        end
+
+        # Set the agents password to the one we generated in this initialization
+        def update_agent_config(config, password)
+          text = File.read(config)
+          File.open(config, 'w') do |f|
+            f.write(text.gsub(/password.*/, "password '#{password}'"))
+          end
+        end
+
+        # Return the fully qualified domain name for this machine. This is used
+        # to determine the agent's JID.
+        def fqdn
+          require 'ohai'
+          system = Ohai::System.new
+          system.require_plugin('os')
+          system.require_plugin('hostname')
+          system.fqdn.downcase
+        end
+
+        # Change the config file to contain the values we generated or validated.
+        def update_config(config)
+# FIXME vines.local
+          orig_config = """
+  host 'vines.local' do
+    cross_domain_messages false
+    private_storage false
+    storage 'fs' do
+      dir 'data/users'
+    end
+    # components 'tea'  => 'secr3t',
+    #            'cake' => 'passw0rd'
+  end"""
+          new_config = """
+  host '#{@domain}' do
+    cross_domain_messages false
+    private_storage true
+    storage 'couchdb' do
+      host '#{@couch_server}'
+      port #{@couch_port}
+      database '#{@couch_db}'
+      tls false
+      username ''
+      password ''
+    end
+    components 'vines'  => '#{@component_password}'
+  end"""
+          text = File.read(config)
+          text = text.gsub(orig_config, new_config)
+          text = text.gsub("host 'vines.wonderland.lit'", "host 'vines.#{@domain}'")
+          text = text.gsub("secr3t", "#{@component_password}")
+          text = text.gsub("host 'localhost'", "host '#{@couch_server}'")
+          text = text.gsub("port 5984", "port #{@couch_port}")
+          text = text.gsub("database 'xmpp'", "database '#{@couch_db}'")
+          File.open(config, 'w') do |f|
+            f.write(text)
+          end
+        end
+      end
+    end
+  end
+end
+
+

data/lib/vines/services/command/restart.rb

@@ -0,0 +1,14 @@
+# encoding: UTF-8
+
+module Vines
+  module Services
+    module Command
+      class Restart
+        def run(opts)
+          Stop.new.run(opts)
+          Start.new.run(opts)
+        end
+      end
+    end
+  end
+end

data/lib/vines/services/command/start.rb

@@ -0,0 +1,30 @@
+# encoding: UTF-8
+
+module Vines
+  module Services
+    module Command
+      class Start
+        def run(opts)
+          raise 'vines-services [--pid FILE] start' unless opts[:args].size == 0
+          require opts[:config]
+          component = Vines::Services::Component.new(Config.instance)
+          daemonize(opts) if opts[:daemonize]
+          component.start
+        end
+
+        private
+
+        def daemonize(opts)
+          daemon = Vines::Daemon.new(:pid => opts[:pid], :stdout => opts[:log],
+            :stderr => opts[:log])
+          if daemon.running?
+            raise "The vines service is running as process #{daemon.pid}"
+          else
+            puts "The vines service has started"
+            daemon.start
+          end
+        end
+      end
+    end
+  end
+end

data/lib/vines/services/command/stop.rb

@@ -0,0 +1,20 @@
+# encoding: UTF-8
+
+module Vines
+  module Services
+    module Command
+      class Stop
+        def run(opts)
+          raise 'vines-services [--pid FILE] stop' unless opts[:args].size == 0
+          daemon = Vines::Daemon.new(:pid => opts[:pid])
+          if daemon.running?
+            daemon.stop
+            puts 'The vines service has been shutdown'
+          else
+            puts 'The vines service is not running'
+          end
+        end
+      end
+    end
+  end
+end

data/lib/vines/services/command/views.rb

@@ -0,0 +1,26 @@
+# encoding: UTF-8
+
+module Vines
+  module Services
+    module Command
+      class Views
+        def run(opts)
+          raise 'vines-services views <domain>' unless opts[:args].size == 1
+          require opts[:config]
+          domain = opts[:args].first
+          unless host = Config.instance.vhosts[domain]
+            raise "#{domain} virtual host not found in conf/config.rb"
+          end
+          unless host.storage.respond_to?(:create_views)
+            raise "CouchDB storage not configured for #{domain} virtual host"
+          end
+          begin
+            host.storage.create_views
+          rescue Exception => e
+            raise "View creation failed: #{e.message}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/vines/services/component.rb

@@ -0,0 +1,26 @@
+# encoding: UTF-8
+
+module Vines
+  module Services
+    # The main starting point for the Vines Services component process. Starts
+    # the EventMachine processing loop and registers the component with the
+    # configured upstream servers.
+    class Component
+      include Vines::Log
+
+      def initialize(config)
+        @config = config
+      end
+
+      def start
+        log.info('Vines component started')
+        at_exit { log.fatal('Vines component stopped') }
+        EM.epoll
+        EM.kqueue
+        EM.run do
+          @config.hosts.each {|host| host.start }
+        end
+      end
+    end
+  end
+end