hoss 0.0.1 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 33580c829a812c199b0a92fbd6a834383b618f51
-  data.tar.gz: 91c016fcd6cebf246ca1e6d1da23e9b33d63d774
+  metadata.gz: 2fbcffb164bffc8e6678fa05d7d7a9f586ecfa71
+  data.tar.gz: 3ec2425370b1f16df23886a7997542e7ff6db82e
 SHA512:
-  metadata.gz: 1dbd5e399d5bed15c24b0b6cff048af700642ca100ebce9c06631d48d22b74b0a003e4b688450c9009aaee345bfff6cd01113b63e938984a26e03d4f6a8654f5
-  data.tar.gz: 4b9bcadb93bd0987d21dd1656f50b6427564b7f611af652d4a99db3f1542a6afa89b4bf5745ab99c3f981a76bcf1ef203db20dced2645801afb53ea8d837d711
+  metadata.gz: 533ef7c673ff7363f4e3d9c9526ec4e6a348d96f8dcc1301cfb71281e061f3160f0382cb1f03ec859bc4dac80df930f02224d8bdf80971c34606756bb38744bd
+  data.tar.gz: 8c7c9f8eb0dc93c1e7f4e51454ce3ca98830429bff0ca210a04031d58c215d88900bdd8eb5691d92e9f0298443b80dcfd03c7f369cc0f143962c0660738e4666

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2014 Sean Clemmer
+Copyright (c) 2015 Sean Clemmer
 
 Permission to use, copy, modify, and/or distribute this software for any
 purpose with or without fee is hereby granted, provided that the above

data/README.md

@@ -1,3 +1,3 @@
-# Hoss
+# Hoss ![Version](https://img.shields.io/gem/v/hoss.svg?style=flat-square)
 
-Like a hoss.
+Index your Chef Server with Elasticsearch.

data/VERSION

@@ -1 +1 @@
-0.0.1
+0.1.0

data/hoss.gemspec

@@ -14,6 +14,10 @@ Gem::Specification.new do |s|
   s.description = Hoss::SUMMARY + '.'
 
   s.add_runtime_dependency 'thor', '~> 0'
+  s.add_runtime_dependency 'pmap', '~> 1'
+  s.add_runtime_dependency 'slog', '~> 1'
+  s.add_runtime_dependency 'ridley', '~> 4.1'
+  s.add_runtime_dependency 'elasticsearch', '~> 1'
 
   s.files         = `git ls-files`.split("\n")
   s.test_files    = `git ls-files -- test/*`.split("\n")

data/lib/hoss/main.rb

@@ -1,15 +1,64 @@
-require 'thor'
+require 'pmap'
+require 'ridley'
+require 'elasticsearch'
 
 require_relative 'metadata'
+require_relative 'mjolnir'
+require_relative 'reindex'
 
 
+Celluloid.logger = nil
+
 module Hoss
-  class Main < Thor
+  class Main < Mjolnir
 
     desc 'version', 'Echo the application version'
     def version
       puts VERSION
     end
 
+
+    desc 'art', 'View the application art'
+    def art
+      puts "\n%s\n" % ART
+    end
+
+
+    desc 'reindex', '(Re)index Chef server objects into Elasticsearch'
+    option :knife_path, \
+      type: :string,
+      aliases: %w[ -k ],
+      desc: 'Path to a knife for Chef server to index'
+    option :elasticsearch_host, \
+      type: :string,
+      aliases: %w[ -e ],
+      desc: 'Set Elasticsearch host to connect client',
+      default: 'localhost:9200'
+    option :index_name, \
+      type: :string,
+      aliases: %w[ -i ],
+      desc: 'Set Elasticsearch index name for object documents',
+      default: 'chef'
+    option :chef_resources, \
+      type: :string,
+      aliases: %w[ -r ],
+      desc: 'Comma-separated list of Chef resources to index',
+      default: 'user,cookbook,data_bag,environment,role,client,node'
+    include_common_options
+    def reindex
+      ridley = Ridley.from_chef_config options.knife
+
+      elasticsearch = Elasticsearch::Client.new \
+        host: options.elasticsearch_host,
+        reload_connections: true,
+        reload_on_failure: true,
+        trace: options.trace?,
+        log: options.trace? ? log : nil
+
+      Reindex.new options.merge \
+        ridley: ridley,
+        elasticsearch: elasticsearch,
+        logger: log
+    end
   end
 end

data/lib/hoss/metadata.rb

@@ -4,7 +4,7 @@ module Hoss
   VERSION  = File.read(File.join(File.dirname(__FILE__), '..', '..', 'VERSION')).strip
 
   # We don't really do all that much, be humble
-  SUMMARY  = 'Like a hoss'
+  SUMMARY  = 'Index your Chef Server with Elasticsearch'
 
   # Your benevolent dictator for life
   AUTHOR   = 'Sean Clemmer'

data/lib/hoss/mjolnir.rb

@@ -0,0 +1,56 @@
+require 'slog'
+require 'thor'
+
+
+# Thor's hammer! Like Thor with better logging
+class Mjolnir < Thor
+
+  # Common options for Thor commands
+  COMMON_OPTIONS = {
+    log: {
+      type: :string,
+      aliases: %w[ -l ],
+      desc: 'Log to file instead of STDOUT',
+      default: ENV['HOSS_LOG'] || nil
+    },
+    debug: {
+      type: :boolean,
+      aliases: %w[ -v ],
+      desc: 'Enable DEBUG-level logging',
+      default: ENV['HOSS_DEBUG'] || false
+    },
+    trace: {
+      type: :boolean,
+      aliases: %w[ -z ],
+      desc: 'Enable TRACE-level logging',
+      default: ENV['HOSS_TRACE'] || false
+    }
+  }
+
+  # Decorate Thor commands with the options above
+  def self.include_common_options
+    COMMON_OPTIONS.each do |name, spec|
+      option name, spec
+    end
+  end
+
+
+  no_commands do
+
+    # Construct a Logger given the command-line options
+    def log
+      return @logger if defined? @logger
+      level = :info
+      level = :debug if options.debug?
+      level = :trace if options.trace?
+      device = options.log || $stderr
+      pretty = device.tty? rescue false
+      @logger = Slog.new \
+        out: device,
+        level: level,
+        colorize: pretty,
+        prettify: pretty
+    end
+
+  end
+end

data/lib/hoss/reindex.rb

@@ -0,0 +1,127 @@
+module Hoss
+  class Reindex
+    attr_reader :resources, :index, :es, :ridley, :log
+
+    def initialize opts
+      @resources = opts[:chef_resources]
+      @index     = opts[:index_name]
+      @es        = opts[:elasticsearch]
+      @ridley    = opts[:ridley]
+      @log       = opts[:logger]
+
+      log.debug \
+        event: 'reindex',
+        opts: opts
+
+      reindex_started = Time.now
+
+      reindex
+
+      log.info \
+        event: 'reindexed',
+        elapsed: Time.now - reindex_started
+    end
+
+
+
+  private
+
+
+    def reindex
+      resources.split(',').each do |resource|
+        started = Time.now
+        log.debug event: 'load_resource', resource: resource
+
+        ops = load_resource resource
+
+        log.info \
+          event: 'loaded_resource',
+          resource: resource,
+          elapsed: Time.now - started
+
+
+        started = Time.now
+        log.debug event: 'index_resource', resource: resource
+
+        index_resource resource, ops
+
+        log.info \
+          event: 'indexed_resource',
+          resource: resource,
+          elapsed: Time.now - started
+      end
+    end
+
+
+
+    def load_resource resource
+      ridley.send(resource.to_sym).all.pmap(8) do |r|
+        r = if resource == 'cookbook'
+          { 'name' => r[0], 'versions' => r[1] }
+        else
+          deep_hash r.reload
+        end
+
+        r['type'] = r['chef_type'] || resource
+        r['id'] = r['name']
+
+        raise if r['id'].nil?
+
+        log.trace event: 'reload_object', object: r
+
+        {
+          index: {
+            _index: index,
+            _type: r['type'],
+            _id: r['id'],
+            data: r
+          }
+        }
+      end
+    end
+
+
+
+    def index_resource resource, ops, bulk_size=500
+      while body = ops.shift(bulk_size)
+        break if body.empty?
+
+        begin
+          log.trace \
+            event: 'bulk_index',
+            response: es.bulk(body: body),
+            resource: resource,
+            size: body.size
+
+          log.debug \
+            event: 'index_resource',
+            resource: resource,
+            size: body.size
+
+        rescue
+          log.error \
+            event: 'bulk_index_failed',
+            resource: resource,
+            size: body.size,
+            body: body.inspect
+        end
+      end
+    end
+
+
+
+    def deep_hash obj
+      if obj.respond_to? :to_hash
+        obj = obj.to_hash
+        obj.keys.each do |k|
+          obj[k] = deep_hash obj[k]
+        end
+        obj
+      else
+        obj
+      end
+    end
+
+
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: hoss
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.1.0
 platform: ruby
 authors:
 - Sean Clemmer
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-12-24 00:00:00.000000000 Z
+date: 2015-03-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -24,7 +24,63 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
-description: Like a hoss.
+- !ruby/object:Gem::Dependency
+  name: pmap
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+- !ruby/object:Gem::Dependency
+  name: slog
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+- !ruby/object:Gem::Dependency
+  name: ridley
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.1'
+- !ruby/object:Gem::Dependency
+  name: elasticsearch
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+description: Index your Chef Server with Elasticsearch.
 email: sclemmer@bluejeans.com
 executables:
 - hoss
@@ -42,6 +98,8 @@ files:
 - lib/hoss.rb
 - lib/hoss/main.rb
 - lib/hoss/metadata.rb
+- lib/hoss/mjolnir.rb
+- lib/hoss/reindex.rb
 homepage: https://github.com/sczizzo/hoss
 licenses:
 - ISC
@@ -62,8 +120,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.4.5
 signing_key: 
 specification_version: 4
-summary: Like a hoss
+summary: Index your Chef Server with Elasticsearch
 test_files: []