puppetdb_query 0.0.3

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 483f7ff2212fc5b4b01df7c5e743848648e7796e
+  data.tar.gz: 86cecf8ac71a8ee6281f351c76a4ac10460def0e
+SHA512:
+  metadata.gz: 830b95936dd945d8540ee67a643d31faca45f4454b37eb2f2983809bed15d8e7594182fe29b84eee1e33977e30e97d45799dd0403752a146b32e61ce0c5321e7
+  data.tar.gz: 03b48691e946b2ecf0fc129e7a78a57c16f75c0d7fb267a843e8dc9b502d64cc80a5785a17089bae320c2b1d717171aa26058331c521c680a927b1c0d68b5eca

data/.gitignore

@@ -0,0 +1,13 @@
+/.bundle/
+/.idea/
+/.yardoc
+/Gemfile.lock
+/*.gem
+/tester.rb
+/test/
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.rubocop.yml

@@ -0,0 +1,102 @@
+# This yaml file describes which files are excluded in rubocop run.
+# It can be in the project home directory or in the $HOME folder.
+
+# Common configuration.
+AllCops:
+  Include:
+    - 'lib/**/'
+    - 'exe/*'
+
+  Exclude:
+    - 'scripts/**/*'
+    - 'vendor/**/*'
+    - 'bin/**/*'
+    - 'bundle/**/*'
+    - 'local-gems/**/*'
+    - '**/*.sh'
+    - '**/Gemfile'
+    - 'tester.rb'
+    - 'test/**/*'
+
+# --- XXXLength-Section --------------------------------------------------------------------
+# too long lines, methods and classes are annoying,
+LineLength:
+  Enabled: true
+  Max: 100
+
+MethodLength:
+  Enabled: true
+  Max: 35
+
+Metrics/AbcSize:
+  Max: 40
+
+ClassLength:
+  Enabled: true
+  Max: 140
+
+# --- Style Cops - Section -----------------------------------------------------------------
+# Don't be so dogmatic about Hash-Style! Both are fine for us
+HashSyntax:
+  Enabled: true
+
+# From Ruby 2.x on there is no need for this anymore, so why bothering now?
+Encoding:
+  Enabled: false
+
+# Ensable following message: Documentation: Missing top-level class documentation comment.
+Documentation:
+  Enabled: true
+
+# check filename conventions
+FileName:
+  Enabled: true
+
+# this 3-digit thing for portnumbers? oh, come on!
+NumericLiterals:
+  Enabled: false
+
+# ok, one should avoid global vars, but from time to time we need them
+Style/GlobalVars:
+  Enabled: true
+
+Style/RegexpLiteral:
+  Enabled: true
+
+Style/AlignParameters:
+  EnforcedStyle: "with_fixed_indentation"
+
+Style/BracesAroundHashParameters:
+  EnforcedStyle: "context_dependent"
+
+Style/EachWithObject:
+  Enabled: false
+
+# we now the special global variables by heart
+Style/SpecialGlobalVars:
+  Enabled: false
+
+# we don't care about quoting style
+Style/StringLiterals:
+  Enabled: false
+
+# for easier line moving
+Style/TrailingCommaInLiteral:
+  Enabled: false
+
+
+# --- Complexity - Section -----------------------------------------------------------------
+# as old McCabe says:
+#
+# Cyclomatic Complexity 	Risk Evaluation...
+# 1-10 	                  A simple module without much risk
+# 11-20 	                A more complex module with moderate risk
+# 21-50 	                A complex module of high risk
+# 51 and greater 	        An untestable program of very high risk
+CyclomaticComplexity:
+  Max: 10
+
+# Lint-Section -----------------------------------------------------------------------------
+# what is soooo bad about blablubb.match /..../ compared to blablubb.match(/..../)?
+Lint/AmbiguousRegexpLiteral:
+  Enabled: true

data/.travis.yml

@@ -0,0 +1,19 @@
+sudo: false
+language: ruby
+cache: bundler
+rvm:
+  - 1.9.3
+  - 2.0.0
+  - 2.1.9
+  - 2.2.5
+  - 2.3.1
+  - jruby-19mode
+  - jruby-head
+  - rbx
+  - rbx-19mode
+matrix:
+  fast_finish: true
+  allow_failures:
+    - rvm: jruby-head
+    - rvm: rbx
+    - rvm: rbx-19mode

data/Gemfile

@@ -0,0 +1,10 @@
+source 'https://rubygems.org'
+
+gemspec
+
+# only for local testing but not needed for spec tests
+group :test do
+  gem 'ruby-puppetdb', '=1.5.3'  if RUBY_VERSION !~ /^1\./
+  gem 'puppet',        '=3.8.7'  if RUBY_VERSION !~ /^1\./
+  gem "rubocop",       '=0.39.0' if RUBY_VERSION =~ /^1\./
+end

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Michael Meyling
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,64 @@
+# puppetdb_query - query puppetdb data from other sources
+
+Just store and update your puppet facts also in another database and query nodes or facts from that other database.
+This can speed up your queries enormously and reduce the load on your puppet database.
+
+## General
+
+The puppet database schema is not designed for complicated queries on numerous nodes. Here we provide
+an implementation for storing and querying node facts in a mongodb.  
+
+You must simply establish a sync job to read the data from your puppetdb and write it to a mongodb.
+
+Currently the implementation supports only puppetdb api V 4.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'puppetdb_query'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install puppetdb_query 
+
+## Usage within program
+
+```ruby
+require "mongo"
+require "puppetdb_query"
+require "pp"
+
+MONGO_HOSTS = ['mongo.myhost.org:27017']
+MONGO_OPTIONS = { database: 'puppetdb', user: 'ops', password: 'very secret' }
+
+pm = PuppetDBQuery::MongoQuery.new(MONGO_HOSTS, MONGO_OPTIONS)
+pm.nodes({processorcount: '4', lvm_support: true})
+pm.facts({processorcount: '4', lvm_support: true}, ["macaddress", "operatingsystem"])
+
+mongo = PuppetDBQuery::ToMongo.new
+query = mongo.query("processorcount='4' and lvm_support=true")
+pp query
+pm.nodes(query)
+pm.facts(query, ["macaddress", "operatingsystem"])
+```
+
+## Development
+
+After checking out the repo, run `bundle install` to install dependencies.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release` to create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+1. Fork it ( https://github.com/m-31/puppetdb_query/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,19 @@
+require 'rake'
+
+require "rubocop/rake_task"
+require 'rspec/core/rake_task'
+
+desc "Run RuboCop on the lib directory"
+RuboCop::RakeTask.new(:rubocop) do |task|
+  task.formatters = ["fuubar"]
+  task.options = ["-D"]
+  task.options = task.options + ["--fail-level", "E"] if RUBY_VERSION =~ /^1\./
+  task.fail_on_error = true
+end
+
+RSpec::Core::RakeTask.new(:spec) do |t|
+  t.pattern = 'spec/**/*_spec.rb'
+end
+
+task :default => ["spec", "rubocop"]
+

data/lib/puppetdb_query/logging.rb

@@ -0,0 +1,45 @@
+require 'logger'
+
+module PuppetDBQuery
+  # for logger access just include this module
+  module Logging
+    class << self
+      attr_writer :logger
+
+      def logger
+        unless @logger
+          @logger = Logger.new($stdout)
+          @logger.level = (ENV['LOG_LEVEL'] || Logger::DEBUG).to_i
+        end
+        @logger
+      end
+    end
+
+    # addition
+    def self.included(base)
+      # rubocop:disable Lint/NestedMethodDefinition
+      class << base
+        def logger
+          # :nocov:
+          Logging.logger
+          # :nocov:
+        end
+
+        def logger=(logger)
+          # :nocov:
+          Logging.logger = logger
+          # :nocov:
+        end
+      end
+      # rubocop:enable Lint/NestedMethodDefinition
+    end
+
+    def logger
+      Logging.logger
+    end
+
+    def logger=(logger)
+      Logging.logger = logger
+    end
+  end
+end

data/lib/puppetdb_query/mongodb.rb

@@ -0,0 +1,134 @@
+require 'time'
+
+require_relative "logging"
+
+module PuppetDBQuery
+  # access nodes and their facts from mongo database
+  class MongoDB
+    include Logging
+    attr_reader :connection
+    attr_reader :nodes_collection
+    attr_reader :nodes_properties_collection
+    attr_reader :meta_collection
+
+    # initialize access to mongodb
+    #
+    # You might want to adjust the logging level, for example:
+    #   ::Mongo::Logger.logger.level = logger.level
+    #
+    # @param connection        mongodb connection, should already be switched to correct database
+    # @param nodes             symbol for collection that contains nodes with their facts
+    # @param nodes_properties  symbol for collection for nodes with their update timestamps
+    # @param meta              symbol for collection with update metadata
+    def initialize(connection, nodes = :nodes, nodes_properties = :nodes_properties, meta = :meta)
+      @connection = connection
+      @nodes_collection = nodes
+      @nodes_propeties_collection = nodes_properties
+      @meta_collection = meta
+    end
+
+    # get node names that fulfill given mongodb query
+    def query_nodes(query)
+      collection = connection[nodes_collection]
+      collection.find(query).batch_size(999).projection(_id: 1).map { |k| k[:_id] }
+    end
+
+    # get nodes and their facts that fulfill given mongodb query
+    def query_facts(query, facts)
+      fields = Hash[facts.collect { |fact| [fact.to_sym, 1] }]
+      collection = connection[nodes_collection]
+      result = {}
+      collection.find(query).batch_size(999).projection(fields).each do |values|
+        id = values.delete('_id')
+        result[id] = values
+      end
+      result
+    end
+
+    # get all node names
+    def nodes
+      collection = connection[nodes_collection]
+      collection.find.batch_size(999).projection(_id: 1).map { |k| k[:_id] }
+    end
+
+    # get facts for given node name
+    def node_facts(node)
+      collection = connection[nodes_collection]
+      result = collection.find(_id: node).limit(999).batch_size(999).to_a.first
+      result.delete("_id") if result
+      result
+    end
+
+    # get all nodes and their facts
+    def facts
+      collection = connection[nodes_collection]
+      result = {}
+      collection.find.batch_size(999).each do |values|
+        id = values.delete('_id')
+        result[id] = values
+      end
+      result
+    end
+
+    # update or insert facts for given node name
+    def node_update(node, facts)
+      connection[nodes_collection].find(_id: node).replace_one(facts, upsert: true)
+    rescue ::Mongo::Error::OperationFailure => e
+      # mongodb doesn't support keys with a dot
+      # see https://docs.mongodb.com/manual/reference/limits/#Restrictions-on-Field-Names
+      # as a dirty workaround we delete the document and insert it ;-)
+      # The dotted field .. in .. is not valid for storage. (57)
+      raise e unless e.message =~ /The dotted field /
+      connection[nodes_collection].find(_id: node).delete_one
+      connection[nodes_collection].insert_one(facts.merge(_id: node))
+    end
+
+    # delete node data for given node name
+    def node_delete(node)
+      connection[nodes_collection].find(_id: node).delete_one
+    end
+
+    # update node properties
+    def node_properties_update(new_node_properties, ts_begin)
+      collection = connection[nodes_properties_collection]
+      old_names = collection.find.batch_size(999).projection(_id: 1).map { |k| k[:_id] }
+      delete = old_names - new_node_properties.keys
+      collection.insert_many(nodes_properties.map { |k, v| v.dup.tap { v[:_id] = k } })
+      collection.delete_many(_id: { '$in' => delete })
+      ts_end = Time.iso8601(Time.now)
+      connection[meta_collection].find_one_and_update(
+        {},
+        {
+          '$set' => {
+            last_node_properties_update: {
+              ts_begin: ts_begin,
+              ts_end:   ts_end
+            }
+          }
+        },
+        { upsert: true }
+      )
+    end
+
+    # update or insert timestamps for given fact update method
+    def meta_fact_update(method, ts_begin, ts_end)
+      connection[meta_collection].find_one_and_update(
+        {},
+        {
+          '$set' => {
+            last_fact_update: {
+              ts_begin: ts_begin,
+              ts_end:   ts_end,
+              method:   method
+            },
+            method => {
+              ts_begin: ts_begin,
+              ts_end:   ts_end
+            }
+          }
+        },
+        { upsert: true }
+      )
+    end
+  end
+end

data/lib/puppetdb_query/operator.rb

@@ -0,0 +1,38 @@
+require_relative "tokenizer"
+
+module PuppetDBQuery
+  # operator with priority and representation information
+  class Operator
+    attr_reader :symbol
+    attr_reader :infix
+    attr_reader :priority
+    attr_reader :minimum
+    attr_reader :maximum
+    attr_reader :string
+
+    def initialize(symbol, infix, priority, minimum, maximum = nil)
+      @symbol = symbol
+      @infix = infix
+      @priority = priority
+      @minimum = minimum
+      @maximum = maximum
+      @string = Tokenizer.symbol_to_string(symbol)
+    end
+
+    def infix?
+      infix
+    end
+
+    def prefix?
+      !infix
+    end
+
+    def ==(other)
+      other.class == self.class && other.symbol == symbol
+    end
+
+    def to_s
+      @string
+    end
+  end
+end

data/lib/puppetdb_query/parser.rb

@@ -0,0 +1,186 @@
+require_relative "operator"
+require_relative "term"
+require_relative "tokenizer"
+require_relative "logging"
+
+module PuppetDBQuery
+  # parse a puppetdb query string into #PuppetDBQuery::Term s
+  class Parser
+    include Logging
+
+    def self.parse(puppetdb_query)
+      Parser.new.parse(puppetdb_query)
+    end
+
+    # these are the operators we understand
+    # rubocop:disable Style/ExtraSpacing
+    AND       = Operator.new(:and,       true, 100, 2)
+    OR        = Operator.new(:or,        true,  90, 2)
+    NOT       = Operator.new(:not,       false,  1, 1, 1)
+    EQUAL     = Operator.new(:equal,     true, 200, 2, 2)
+    NOT_EQUAL = Operator.new(:not_equal, true, 200, 2, 2)
+    MATCH     = Operator.new(:match,     true, 200, 2, 2)
+    # rubocop:enable Style/ExtraSpacing
+
+    # map certain symbols (we get them from a tokenizer) to our operators
+    OPERATORS = {
+      AND.symbol       =>  AND,
+      OR.symbol        =>  OR,
+      NOT.symbol       =>  NOT,
+      EQUAL.symbol     =>  EQUAL,
+      NOT_EQUAL.symbol =>  NOT_EQUAL,
+      MATCH.symbol     =>  MATCH,
+    }.freeze
+
+    attr_reader :symbols  # array of symbols
+    attr_reader :position # current parsing position in array of symbols
+
+    # parse query and get resulting array of PuppetDBQuery::Term s
+    def parse(query)
+      @symbols = Tokenizer.symbols(query)
+      @position = 0
+      r = []
+      r << read_maximal_term(0) until empty?
+      r
+    end
+
+    private
+
+    # Reads next maximal term. The following input doesn't make the term ore complete.
+    # Respects the priority of operators by comparing it to the given value.
+    def read_maximal_term(priority)
+      return nil if empty?
+      first = read_minimal_term
+      term = add_next_infix_terms(priority, first)
+      logger.debug "read maximal term: #{term}"
+      term
+    end
+
+    # Read next following term. This is a complete term but some infix operator
+    # or some terms for an infix operator might follow.
+    # rubocop:disable Metrics/PerceivedComplexity
+    def read_minimal_term
+      term = nil
+      operator = get_operator
+      if operator
+        error("'#{operator}' is no prefix operator") unless operator.prefix?
+        read_token
+        term = Term.new(operator)
+        arg = read_maximal_term(operator.priority)
+        term.add(arg)
+        logger.debug "read_minimal_term: #{term}"
+        return term
+      end
+      # no prefix operator found
+      token = get_token
+      if token == :begin
+        read_token
+        term = read_maximal_term(0)
+        error "'#{Tokenizer.symbol_to_string(:end)}' expected " unless read_token == :end
+      elsif token == :list_begin
+        read_token
+        term = read_maximal_term(0)
+        error "'#{Tokenizer.symbol_to_string(:list_end)}' expected " unless read_token == :list_end
+      else
+        error("no operator #{get_operator} expected here") if get_operator
+        token = read_token
+        logger.debug "atom found: #{token}"
+        term = token
+      end
+      logger.debug "read minimal term: #{term}"
+      term
+    end
+    # rubocop:enable Metrics/PerceivedComplexity
+
+    # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity
+    # rubocop:disable Metrics/MethodLength,Metrics/PerceivedComplexity
+    def add_next_infix_terms(priority, first)
+      old_operator = nil
+      term = first
+      loop do
+        # we expect an infix operator
+        operator = get_operator
+        logger.debug "we found operator '#{operator}'" if operator
+        if operator.nil? || operator.prefix? || operator.priority <= priority
+          logger.debug "'#{operator}' is prefex '#{operator && operator.prefix?}' or has less" \
+                       " priority #{operator && operator.priority} than #{priority}"
+          logger.debug "get_next_infix_terms: #{term}"
+          return term
+        end
+        if old_operator.nil? || old_operator.priority >= operator.priority
+          # old operator has not less priority
+          read_token
+          new_term = read_maximal_term(operator.priority)
+          error("to few arguments for operator '#{operator}'") if new_term.nil?
+          logger.debug "is '#{old_operator}' == '#{operator}' : #{old_operator == operator}"
+          if old_operator == operator
+            if operator.maximum && term.args.size + 1 >= operator.maximum
+              raise "to much arguments for operator '#{operator}'"
+            end
+            term.add(new_term)
+          else
+            also_new_term = Term.new(operator)
+            also_new_term.add(term)
+            also_new_term.add(new_term)
+            term = also_new_term
+          end
+        else
+          # old operator has less priority
+          new_term = read_maximal_term(operator.priority)
+          error("to few arguments for operator '#{operator}'") if new_term.nil?
+          also_new_term = Term.new(operator)
+          also_new_term.add(term)
+          also_new_term.add(new_term)
+          term = also_new_term
+        end
+        old_operator = operator
+      end
+    end
+    # rubocop:enable Metrics/AbcSize,Metrics/CyclomaticComplexity
+    # rubocop:enable Metrics/MethodLength,Metrics/PerceivedComplexity
+
+    # rubocop:disable Style/AccessorMethodName
+    def get_operator
+      OPERATORS[get_token]
+    end
+    # rubocop:enable Style/AccessorMethodName
+
+    def read_token
+      return nil if empty?
+      token = symbols[position]
+      @position += 1
+      token
+    end
+
+    def empty?
+      position >= symbols.size
+    end
+
+    # rubocop:disable Style/AccessorMethodName
+    def get_token
+      return nil if empty?
+      symbols[position]
+    end
+    # rubocop:enable Style/AccessorMethodName
+
+    def error(message)
+      length = Tokenizer.query(symbols[0..position]).size
+      raise "parsing query failed\n#{message}\n\n#{Tokenizer.query(symbols)}\n#{' ' * length}^"
+    end
+  end
+end
+
+if $0 == __FILE__
+  require "pp"
+  query = "facts=-7.4E1 and fucts=8 and fits=true or application_vertical='ops' and" \
+          " (application_group=\"live\"or application_group='prelive-production')"
+  puts query
+  query = PuppetDBQuery::Tokenizer.idem(query)
+  puts query
+  query = PuppetDBQuery::Tokenizer.idem(query)
+  puts query
+
+  parser = PuppetDBQuery::Parser.new
+  tree = parser.parse(query)
+  pp tree
+end