puppetdb_query 0.0.3

data/lib/puppetdb_query/puppetdb.rb

@@ -0,0 +1,90 @@
+require 'net/http'
+require 'json'
+
+require_relative 'logging'
+
+module PuppetDBQuery
+  # access puppetdb data
+  class PuppetDB
+    include Logging
+
+    NODES = "/v4/nodes".freeze
+    FACTS = "/v4/facts".freeze
+
+    def initialize(host = HOST, port = 443, protocol = "https", nodes = NODES, facts = FACTS)
+      @nodes_url = "#{protocol}://#{host}:#{port}#{nodes}"
+      @facts_url = "#{protocol}://#{host}:#{port}#{facts}"
+      @lock = Mutex.new
+    end
+
+    # get array of node names
+    def nodes
+      # TODO: perhaps we have to ignore entries without "deactivated": null?
+      # TODO: in '/v3/nodes' we must take 'name'
+      api_nodes.map { |data| data['certname'] }
+    end
+
+    # get array of node names
+    def nodes_properties
+      result = {}
+      api_nodes.each do |data|
+        next if data['deactivated']
+        # TODO: in '/v3/nodes' we must take 'name'
+        name = data['certname']
+        values = data.dup
+        %w(deactivated certname).each { |key| values.delete(key) }
+        result[name] = values
+      end
+      result
+    end
+
+    # get all nodes that have updated facts
+    def nodes_update_facts_since(timestamp)
+      ts = (timestamp.is_a?(String) ? Time.iso8601(ts) : timestamp)
+      nodes_properties.delete_if do |_k, data|
+        # TODO: in '/v3/nodes' we must take 'facts_timestamp'
+        !data["facts-timestamp"] || Time.iso8601(data["facts-timestamp"]) < ts
+      end.keys
+    end
+
+    # get hash of facts for given node name
+    def node_facts(node)
+      json = get_json("#{@nodes_url}/#{node}/facts", 10)
+      return nil if json.include?("error")
+      Hash[json.map { |data| [data["name"], data["value"]] }]
+    end
+
+    # get all nodes with all facts
+    def facts
+      json = get_json(@facts_url, 60)
+      result = {}
+      json.each do |fact|
+        data = result[fact["certname"]]
+        result[fact["certname"]] = data = {} unless data
+        data[fact["name"]] = fact["value"]
+      end
+      result
+    end
+
+    def api_nodes
+      get_json(@nodes_url, 10)
+    end
+
+    private
+
+    def get_json(url, timeout)
+      @lock.synchronize do
+        logger.info "get json from #{url}"
+        uri = URI.parse(url)
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = uri.scheme == 'https'
+        http.read_timeout = timeout
+        request = Net::HTTP::Get.new(uri.request_uri)
+        request['Accept'] = "application/json"
+        response = http.request(request)
+        logger.info "  got #{response.body.size} characters from #{url}"
+        JSON.parse(response.body)
+      end
+    end
+  end
+end

data/lib/puppetdb_query/term.rb

@@ -0,0 +1,31 @@
+module PuppetDBQuery
+  # represent a term containing an operator and arguments
+  class Term
+    attr_reader :operator
+    attr_reader :args
+
+    def initialize(operator)
+      @operator = operator
+      @args = []
+    end
+
+    def add(*arg)
+      @args += arg
+      self
+    end
+
+    def ==(other)
+      other.class == self.class && other.operator == operator && other.args == args
+    end
+
+    def to_s
+      if operator.prefix?
+        "#{operator}(#{args.join(', ')})"
+      elsif operator.infix?
+        "(#{args.join(" #{operator} ")})"
+      else
+        raise "unkown representation for operator: #{operator}"
+      end
+    end
+  end
+end

data/lib/puppetdb_query/to_mongo.rb

@@ -0,0 +1,52 @@
+require_relative "parser"
+require_relative "logging"
+
+module PuppetDBQuery
+  # convert puppetdb query into mongodb query
+  class ToMongo
+    include Logging
+
+    def query(string)
+      logger.info "transfer following string into mongo query:"
+      logger.info(string)
+      terms = Parser.parse(string)
+      mongo_query = query_term(terms[0])
+      logger.info "resulting mongo query:"
+      logger.info mongo_query.inspect
+      mongo_query
+    end
+
+    private
+
+    # rubocop:disable Metrics/PerceivedComplexity
+    def query_term(term)
+      # rubocop:disable Style/GuardClause
+      if term.is_a?(Symbol)
+        return term.to_s
+      elsif term.is_a?(Integer)
+        return "'#{term}'"
+      elsif term.is_a?(TrueClass)
+        return term.to_s
+      elsif !term.is_a?(Term)
+        return "'#{term}'"
+      end
+      # rubocop:enable Style/GuardClause
+      terms = term.args.map { |t| query_term(t) }
+      case term.operator.symbol
+      when :and
+        { :$and => terms }
+      when :or
+        { :$or => terms }
+      when :equal
+        { term.args[0] => term.args[1].to_s }
+      when :not_equal
+        { term.args[0] => { :$ne => term.args[1].to_s } }
+      when :match
+        { term.args[0] => { :$regex => term.args[1].to_s } }
+      else
+        raise "can't handle operator '#{term.operator}' yet"
+      end
+    end
+    # rubocop:enable Metrics/PerceivedComplexity
+  end
+end

data/lib/puppetdb_query/tokenizer.rb

@@ -0,0 +1,185 @@
+require_relative "logging"
+
+module PuppetDBQuery
+  # tokenize puppetdb queries
+  # rubocop:disable Metrics/ClassLength
+  class Tokenizer
+    include Logging
+    include Enumerable
+
+    SINGLE_CHAR_TO_TOKEN = {
+      "!" => :not,
+      "=" => :equal,
+      "(" => :begin,
+      ")" => :end,
+      "[" => :list_begin,
+      "]" => :list_end,
+      "<" => :less,
+      ">" => :greater,
+      "~" => :match,
+      "," => :comma,
+    }.freeze
+
+    DOUBLE_CHAR_TO_TOKEN = {
+      "!=" => :not_equal,
+      "!~" => :not_match,
+      "~>" => :match_array,
+      "<=" => :less_or_equal,
+      ">=" => :greater_or_equal,
+    }.freeze
+
+    STRING_TO_TOKEN = {
+      "not"   => :not,
+      "or"    => :or,
+      "and"   => :and,
+      "in"    => :in,
+      "is"    => :is,
+      "null"  => :null,
+      "true"  => :true,
+      "false" => :false,
+    }.freeze
+
+    LANGUAGE_TOKENS = SINGLE_CHAR_TO_TOKEN.merge(DOUBLE_CHAR_TO_TOKEN).merge(STRING_TO_TOKEN).freeze
+    LANGUAGE_STRINGS = LANGUAGE_TOKENS.invert.freeze
+
+    def self.symbols(query)
+      r = []
+      tokenizer = Tokenizer.new(query)
+      r << tokenizer.next_token until tokenizer.empty?
+      r
+    end
+
+    def self.query(symbols)
+      symbols.map { |v| symbol_to_string(v) }.join(" ")
+    end
+
+    def self.symbol_to_string(s)
+      (LANGUAGE_STRINGS[s] || (s.is_a?(Symbol) ? s.to_s : nil) || s.inspect).to_s
+    end
+
+    def self.idem(query)
+      query(symbols(query))
+    end
+
+    attr_reader :position
+    attr_reader :text
+
+    def initialize(text)
+      @text = text
+      @position = 0
+    end
+
+    def next_token
+      skip_whitespace
+      return nil if empty?
+      read_token
+    end
+
+    def empty?
+      position >= text.size
+    end
+
+    def each
+      yield next_token until empty?
+    end
+
+    private
+
+    def read_token
+      logger.debug "read token"
+      skip_whitespace
+      return nil if empty?
+      s = text[position, 2]
+      if DOUBLE_CHAR_TO_TOKEN.include?(s)
+        increase
+        increase
+        return DOUBLE_CHAR_TO_TOKEN[s]
+      end
+      c = text[position]
+      if SINGLE_CHAR_TO_TOKEN.include?(c)
+        increase
+        return SINGLE_CHAR_TO_TOKEN[c]
+      end
+      case c
+      when /[a-zA-Z]/
+        return read_symbol
+      when "'", '"'
+        return read_quoted
+      when /[-0-9]/
+        return read_number
+      else
+        error("unknown kind of token: '#{c}'")
+      end
+    end
+
+    def read_quoted
+      logger.debug "read quoted"
+      skip_whitespace
+      q = text[position]
+      increase
+      r = ""
+      while !empty? && (c = text[position]) != q
+        if c == "\\"
+          increase
+          c = text[position] unless empty?
+          case c
+          when 'r'
+            c = "\r"
+          when 'n'
+            c = "\n"
+          when '\''
+            c = "\\"
+          end
+        end
+        r << c
+        increase
+      end
+      error("I expected '#{q}' but I got '#{c}'") if c != q
+      increase
+      logger.debug "resulting string: '#{r}'"
+      r
+    end
+
+    def read_symbol
+      logger.debug "read symbol"
+      skip_whitespace
+      r = ""
+      while !empty? && (c = text[position]) =~ /[-a-zA-Z_0-9]/
+        r << c
+        increase
+      end
+      logger.debug "resulting symbol: '#{r}'"
+      r.to_sym
+    end
+
+    def read_number
+      logger.debug "read number"
+      skip_whitespace
+      r = ""
+      while !empty? && (c = text[position]) =~ /[-0-9\.E]/
+        r << c
+        increase
+      end
+      logger.debug "resulting number: '#{r}'"
+      Integer(r)
+    rescue
+      Float(r)
+    end
+
+    def skip_whitespace
+      # logger.debug "skip whitespace"
+      return if empty?
+      increase until empty? || text[position] !~ /\s/
+    end
+
+    def increase
+      # logger.debug "increase"
+      @position += 1
+      # logger.debug position
+    end
+
+    def error(message)
+      raise "tokenizing query failed\n#{message}\n\n#{text}\n#{' ' * position}^"
+    end
+  end
+end

data/lib/puppetdb_query/updater.rb

@@ -0,0 +1,76 @@
+require_relative "logging"
+
+module PuppetDBQuery
+  # update nodes data from source to destination
+  class Updater
+    include Logging
+
+    attr_reader :source
+    attr_reader :destination
+
+    def initialize(source, destination)
+      @source = source
+      @destination = destination
+    end
+
+    # update by deleting missing nodes and iterating over all nodes and
+    # update or insert facts for each one
+    #
+    # 335.6 seconds: update time for 1561 nodes
+    def update
+      source_nodes = source.nodes
+      destination_nodes = destination.nodes
+      (destination_nodes - source_nodes).each do |node|
+        destination.node_delete(node)
+      end
+      source_nodes.each do |node|
+        begin
+          destination.node_update(node, source.node_facts(node))
+        rescue
+          logging.error $!
+        end
+      end
+    end
+
+    # update by deleting missing nodes and get a complete map of nodes with facts
+    # and update or insert facts for each one
+    #
+    # 166.4 seconds: update time for 1561 nodes
+    def update2
+      source_nodes = source.nodes
+      destination_nodes = destination.nodes
+      (destination_nodes - source_nodes).each do |node|
+        destination.node_delete(node)
+      end
+      complete = source.facts
+      complete.each do |node, facts|
+        begin
+          destination.node_update(node, facts)
+        rescue
+          logging.error $!
+        end
+      end
+    end
+
+    # update by deleting missing nodes and getting a list of nodes
+    # with changed facts, iterate over them and update or insert facts for each one
+    #
+    # update time depends extremly on the number of changed nodes
+    def update3(last_update_timestamp)
+      source_nodes = source.nodes
+      destination_nodes = destination.nodes
+      (destination_nodes - source_nodes).each do |node|
+        destination.node_delete(node)
+      end
+      modified = source.nodes_update_facts_since(last_update_timestamp)
+      modified.each do |node|
+        begin
+          destination.node_update(node, source.node_facts(node))
+        rescue
+          logging.error $!
+        end
+      end
+      modified.size
+    end
+  end
+end

data/lib/puppetdb_query/version.rb

@@ -0,0 +1,3 @@
+module PuppetDBQuery
+  VERSION = "0.0.3".freeze
+end

data/lib/puppetdb_query.rb

@@ -0,0 +1,14 @@
+require_relative "puppetdb_query/logging"
+require_relative "puppetdb_query/operator"
+require_relative "puppetdb_query/term"
+require_relative "puppetdb_query/tokenizer"
+require_relative "puppetdb_query/parser"
+require_relative "puppetdb_query/puppetdb"
+require_relative "puppetdb_query/mongodb"
+require_relative "puppetdb_query/to_mongo"
+require_relative "puppetdb_query/updater"
+require_relative "puppetdb_query/version"
+
+# use puppetdb query language to ask a mongodb for node facts
+module PuppetDBQuery
+end

data/puppetdb_query.gemspec

@@ -0,0 +1,28 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'puppetdb_query/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "puppetdb_query"
+  spec.version       = PuppetDBQuery::VERSION
+  spec.authors       = ["Michael Meyling"]
+  spec.email         = ["search@meyling.com"]
+  spec.summary       = %q{access puppetdb data from other sources}
+  spec.description   = %q{Allow puppetdb access from resources like mongodb.}
+  spec.homepage      = "https://github.com/m-31/puppetdb_query"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+  spec.post_install_message = "  Sitting quietly, doing nothing, spring comes, and grass grows by itself."
+
+  spec.add_dependency             "mongo"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "rubocop" #, "0.43.0" # matches with .rubocop.yml
+  spec.add_development_dependency "easy_diff"
+end

data/spec/puppetdb_query/parser_spec.rb

@@ -0,0 +1,40 @@
+require "spec_helper"
+
+# rubocop:disable Style/SpaceInsideBrackets,Style/MultilineArrayBraceLayout
+# rubocop:disable Style/MultilineMethodCallIndentation,Style/RedundantParentheses
+# rubocop:disable Style/ClosingParenthesisIndentation
+describe PuppetDBQuery::Parser do
+  PARSER_DATA = [
+    [ 'hostname=\'puppetdb-mike-217922\'',
+      [PuppetDBQuery::Term.new(PuppetDBQuery::Parser::EQUAL).add(:hostname, "puppetdb-mike-217922")]
+    ],
+    [ 'disable_puppet = true',
+      [PuppetDBQuery::Term.new(PuppetDBQuery::Parser::EQUAL).add(:disable_puppet, :true)]
+    ],
+    [ 'fqdn~"app-dev" and group=develop and vertical~tracking and cluster_color~BLUE',
+      [PuppetDBQuery::Term.new(PuppetDBQuery::Parser::AND)
+         .add(PuppetDBQuery::Term.new(PuppetDBQuery::Parser::MATCH).add(:fqdn, "app-dev"))
+         .add(PuppetDBQuery::Term.new(PuppetDBQuery::Parser::EQUAL).add(:group, :develop))
+         .add(PuppetDBQuery::Term.new(PuppetDBQuery::Parser::MATCH).add(:vertical, :tracking))
+         .add(PuppetDBQuery::Term.new(PuppetDBQuery::Parser::MATCH).add(:cluster_color, :BLUE))
+      ]
+    ],
+    [ 'a~"A" and g=G or v~t and c~B',
+      [PuppetDBQuery::Term.new(PuppetDBQuery::Parser::OR)
+        .add((PuppetDBQuery::Term.new(PuppetDBQuery::Parser::AND)
+          .add(PuppetDBQuery::Term.new(PuppetDBQuery::Parser::MATCH).add(:a, "A"))
+          .add(PuppetDBQuery::Term.new(PuppetDBQuery::Parser::EQUAL).add(:g, :G))
+         )).add((PuppetDBQuery::Term.new(PuppetDBQuery::Parser::AND)
+          .add(PuppetDBQuery::Term.new(PuppetDBQuery::Parser::MATCH).add(:v, :t))
+          .add(PuppetDBQuery::Term.new(PuppetDBQuery::Parser::MATCH).add(:c, :B))
+         ))
+      ]
+    ],
+  ].freeze
+
+  PARSER_DATA.each do |q, a|
+    it "translates correctly #{q.inspect}" do
+      expect(subject.parse(q)).to eq(a)
+    end
+  end
+end

data/spec/puppetdb_query/to_mongo_spec.rb

@@ -0,0 +1,68 @@
+require "spec_helper"
+
+# rubocop:disable Style/SpaceInsideBrackets,Style/MultilineArrayBraceLayout,Style/IndentArray
+# rubocop:disable Style/MultilineHashBraceLayout
+describe PuppetDBQuery::ToMongo do
+  TO_MONGO_DATA = [
+    [ "hostname='puppetdb-mike-217922'",
+      { hostname: "puppetdb-mike-217922" }
+    ],
+    [ 'disable_puppet = true',
+      { disable_puppet: "true" }
+    ],
+    [ 'fqdn~"app-dev" and group=develop and vertical~tracking and cluster_color~BLUE',
+      { :$and => [
+          { fqdn: { :$regex => "app-dev" } },
+          { group: "develop" },
+          { vertical: { :$regex => "tracking" } },
+          { cluster_color: { :$regex => "BLUE" } }
+        ]
+      }
+    ],
+    [ 'fqdn~"kafka" and group=develop and vertical=tracking',
+      { :$and => [
+          { fqdn: { :$regex => "kafka" } },
+          { group: "develop" },
+          { vertical: "tracking" }
+        ]
+      }
+    ],
+    [ '(group="develop-ci" or group=develop or group=mock) and (operatingsystemmajrelease="6")',
+      { :$and => [
+          { :$or => [
+              { group: "develop-ci" },
+              { group: "develop" },
+              { group: "mock" }
+            ]
+          },
+          { operatingsystemmajrelease: "6" }
+        ]
+      }
+    ],
+    [ "server_type=zoo or server_type='mesos-magr') and group!='infrastructure-ci'",
+      { :$or => [
+          { server_type: "zoo" },
+          { server_type: "mesos-magr" }
+        ]
+      }
+    ],
+    [ "server_type~'mesos-magr' and group='ops-ci' and operatingsystemmajrelease=7 and" \
+      " vmtest_vm!=true and disable_puppet!=true and puppet_artifact_version!=NO_VERSION_CHECK",
+      { :$and => [
+          { server_type: { :$regex => "mesos-magr" } },
+          { group: "ops-ci" },
+          { operatingsystemmajrelease: "7" },
+          { vmtest_vm: { :$ne => "true" } },
+          { disable_puppet: { :$ne => "true" } },
+          { puppet_artifact_version: { :$ne => "NO_VERSION_CHECK" } }
+        ]
+      }
+    ],
+  ].freeze
+
+  TO_MONGO_DATA.each do |q, a|
+    it "translates correctly #{q.inspect}" do
+      expect(subject.query(q)).to eq(a)
+    end
+  end
+end

data/spec/puppetdb_query/tokenizer_spec.rb

@@ -0,0 +1,44 @@
+require "spec_helper"
+
+# rubocop:disable Style/SpaceInsideBrackets,Style/MultilineArrayBraceLayout
+describe PuppetDBQuery::Tokenizer do
+  TOKENIZER_DATA = [
+    [ 'hostname=\'puppetdb-mike-217922\'',
+      [:hostname, :equal, "puppetdb-mike-217922"]
+    ],
+    [ 'disable_puppet = true',
+      [:disable_puppet, :equal, :true]
+    ],
+    [ 'fqdn~"app-dev" and group=develop and vertical~tracking and cluster_color~BLUE',
+      [:fqdn, :match, "app-dev", :and, :group, :equal, :develop, :and, :vertical, :match,
+       :tracking, :and, :cluster_color, :match, :BLUE]
+    ],
+    [ 'fqdn~"kafka" and group=develop and vertical=tracking',
+      [:fqdn, :match, "kafka", :and, :group, :equal, :develop, :and, :vertical, :equal, :tracking]
+    ],
+    [ '(group="develop-ci" or group=develop or group=mock) and (operatingsystemmajrelease="6")',
+      [:begin, :group, :equal, "develop-ci", :or, :group, :equal, :develop, :or, :group, :equal,
+       :mock, :end, :and, :begin, :operatingsystemmajrelease, :equal, "6", :end]
+    ],
+    [ "server_type=zoo or server_type='mesos-magr') and group!='infrastructure-ci'",
+      [:server_type, :equal, :zoo, :or, :server_type, :equal, "mesos-magr", :end, :and, :group,
+       :not_equal, "infrastructure-ci"]
+    ],
+    [ "server_type~'mesos-magr' and group='ops-ci' and operatingsystemmajrelease=7 and" \
+      " vmtest_vm!=true and disable_puppet!=true and puppet_artifact_version!=NO_VERSION_CHECK",
+      [:server_type, :match, "mesos-magr", :and, :group, :equal, "ops-ci", :and,
+       :operatingsystemmajrelease, :equal, 7, :and, :vmtest_vm, :not_equal, :true, :and,
+       :disable_puppet, :not_equal, :true, :and, :puppet_artifact_version, :not_equal,
+       :NO_VERSION_CHECK]
+    ],
+  ].freeze
+
+  TOKENIZER_DATA.each do |q, a|
+    describe "translates correctly #{q.inspect}" do
+      subject { PuppetDBQuery::Tokenizer.new(q) }
+      it "into tokens" do
+        expect(subject.map { |n| n }).to eq(a)
+      end
+    end
+  end
+end

data/spec/puppetdb_query_spec.rb

@@ -0,0 +1,7 @@
+require "spec_helper"
+
+describe PuppetDBQuery do
+  it "has a version number" do
+    expect(PuppetDBQuery::VERSION).not_to be nil
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,14 @@
+$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
+require 'puppetdb_query'
+
+# no logging output during spec tests
+include PuppetDBQuery::Logging
+logger.level = Logger::FATAL
+
+# we want to be able to test protected or private methods
+RSpec.configure do |config|
+  config.before(:each) do
+    described_class.send(:public, *described_class.protected_instance_methods)
+    described_class.send(:public, *described_class.private_instance_methods)
+  end
+end