puppetdb_query 0.0.8 → 0.0.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 91b27838db87cfeca399ae3ca4bcc003e6ccd529
-  data.tar.gz: bbbf8a6880fdf6c4e74b5354767dcb6a7e2c127c
+  metadata.gz: dd4a70cd22ca95133379508256d33bc63c0d922d
+  data.tar.gz: 4c2d190ac3fac79c08bc99786123665d8aacdc75
 SHA512:
-  metadata.gz: b46aa5c022e19434fb95c920f3746a87b25b0ac3cd01eed3e7499a69ecd8135de08bbe71783cc55ec428d6389d1904ffed5f0a89a02eb538ae06a0779b9e4aab
-  data.tar.gz: 4e80609d15bc887be8aaf146793acb989a3243eadca4ed6e8786eadf55cc25ecf62a5e4f9e9b4c19170c17c2fe2a792ff844f7ad2c0eeea7ddb983c14f03ba6c
+  metadata.gz: 5d688015cbd72fc52980ecfc2415a3312a7d83a610cec35c4e41a851e5fddf30093ee8525a987ec1cf085073a5e686863a8cc1eddaf08cb70601ec73ca6ee96d
+  data.tar.gz: bff404d3da3089951c784e6508be8150e667f9b061d53a373ae7d50833aaa04ffbbf56a5bc6bfda5268bbc6646c2a7288bc9daea97097700af6a71542c3ca76a

data/lib/puppetdb_query/parser.rb

@@ -5,6 +5,7 @@ require_relative "logging"
 
 module PuppetDBQuery
   # parse a puppetdb query string into #PuppetDBQuery::Term s
+  # rubocop:disable Metrics/ClassLength
   class Parser
     include Logging
 
@@ -14,12 +15,13 @@ module PuppetDBQuery
 
     # these are the operators we understand
     # rubocop:disable Style/ExtraSpacing
-    AND       = Operator.new(:and,       true,  100, 2)
-    OR        = Operator.new(:or,        true,   90, 2)
-    NOT       = Operator.new(:not,       false, 150, 1, 1)
-    EQUAL     = Operator.new(:equal,     true,  200, 2, 2)
-    NOT_EQUAL = Operator.new(:not_equal, true,  200, 2, 2)
-    MATCH     = Operator.new(:match,     true,  200, 2, 2)
+    AND       = Operator.new(:_and,       true,  100, 2)
+    OR        = Operator.new(:_or,        true,   90, 2)
+    NOT       = Operator.new(:_not,       false, 150, 1, 1)
+    EQUAL     = Operator.new(:_equal,     true,  200, 2, 2)
+    NOT_EQUAL = Operator.new(:_not_equal, true,  200, 2, 2)
+    MATCH     = Operator.new(:_match,     true,  200, 2, 2)
+    IN        = Operator.new(:_in,        true,  200, 2, 2)
     # rubocop:enable Style/ExtraSpacing
 
     # map certain symbols (we get them from a tokenizer) to our operators
@@ -30,6 +32,7 @@ module PuppetDBQuery
       EQUAL.symbol     =>  EQUAL,
       NOT_EQUAL.symbol =>  NOT_EQUAL,
       MATCH.symbol     =>  MATCH,
+      IN.symbol        =>  IN,
     }.freeze
 
     attr_reader :symbols  # array of symbols
@@ -46,10 +49,11 @@ module PuppetDBQuery
 
     private
 
-    # Reads next maximal term. The following input doesn't make the term ore complete.
+    # Reads next maximal term. The following input doesn't make the term more complete.
     # Respects the priority of operators by comparing it to the given value.
     def read_maximal_term(priority)
       return nil if empty?
+      logger.debug "read maximal term (#{priority})"
       first = read_minimal_term
       term = add_next_infix_terms(priority, first)
       logger.debug "read maximal term: #{term}"
@@ -58,31 +62,55 @@ module PuppetDBQuery
 
     # Read next following term. This is a complete term but some infix operator
     # or some terms for an infix operator might follow.
-    # rubocop:disable Metrics/PerceivedComplexity
+    # rubocop:disable Metrics/PerceivedComplexity,Metrics/AbcSize,Metrics/CyclomaticComplexity
+    # rubocop:disable Metrics/MethodLength
     def read_minimal_term
+      logger.debug "read minimal term"
       term = nil
       operator = get_operator
       if operator
         error("'#{operator}' is no prefix operator") unless operator.prefix?
         read_token
         term = Term.new(operator)
-        arg = read_maximal_term(operator.priority)
-        term.add(arg)
+        if operator.maximum > 1
+          error("'#{operator}' is prefix operator with more than one argument," \
+                " that is not supported yet")
+        end
+        if operator.minimum > 0
+          arg = read_maximal_term(operator.priority)
+          error("prefix operator '#{operator}' got no argument") if arg.nil?
+          term.add(arg)
+        end
         logger.debug "read_minimal_term: #{term}"
         return term
       end
       # no prefix operator found
       token = get_token
-      if token == :begin
+      if token == :_begin
+        logger.debug "read ()"
         read_token
         term = read_maximal_term(0)
-        error "'#{Tokenizer.symbol_to_string(:end)}' expected " unless read_token == :end
-      elsif token == :list_begin
+        error "'#{Tokenizer.symbol_to_string(:_end)}' expected " unless get_token == :_end
         read_token
-        term = read_maximal_term(0)
-        error "'#{Tokenizer.symbol_to_string(:list_end)}' expected " unless read_token == :list_end
+      elsif token == :_list_begin
+        logger.debug "read []"
+        read_token
+        args = []
+        loop do
+          term = read_maximal_term(0)
+          if get_token != :_list_end && get_token != :_comma
+            error "'#{Tokenizer.symbol_to_string(:_list_end)}' or" \
+                  " '#{Tokenizer.symbol_to_string(:_comma)}' expected"
+          end
+          args << term if term
+          break if read_token == :_list_end
+        end
+        term = args
       else
         error("no operator #{get_operator} expected here") if get_operator
+        if Tokenizer::LANGUAGE_STRINGS.include?(token) && ![:true, :false].include?(token)
+          error("that was not expected here: '#{Tokenizer.symbol_to_string(token)}'")
+        end
         token = read_token
         logger.debug "atom found: #{token}"
         term = token
@@ -90,7 +118,8 @@ module PuppetDBQuery
       logger.debug "read minimal term: #{term}"
       term
     end
-    # rubocop:enable Metrics/PerceivedComplexity
+    # rubocop:enable Metrics/PerceivedComplexity,Metrics/AbcSize,Metrics/CyclomaticComplexity
+    # rubocop:enable Metrics/MethodLength
 
     # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity
     # rubocop:disable Metrics/MethodLength,Metrics/PerceivedComplexity
@@ -100,10 +129,14 @@ module PuppetDBQuery
       loop do
         # we expect an infix operator
         operator = get_operator
-        logger.debug "we found operator '#{operator}'" if operator
+        logger.debug "we found operator '#{operator}'" unless operator.nil?
         if operator.nil? || operator.prefix? || operator.priority <= priority
-          logger.debug "'#{operator}' is prefex '#{operator && operator.prefix?}' or has less" \
-                       " priority #{operator && operator.priority} than #{priority}"
+          logger.debug "operator '#{operator}' is nil" if operator.nil?
+          logger.debug "operator '#{operator}' is prefex" if !operator.nil? && operator.prefix?
+          if !operator.nil? && operator.priority <= priority
+            logger.debug "operator '#{operator}' has less priority #{operator.priority}" \
+                         " than #{priority}"
+          end
           logger.debug "get_next_infix_terms: #{term}"
           return term
         end
@@ -164,10 +197,11 @@ module PuppetDBQuery
     # rubocop:enable Style/AccessorMethodName
 
     def error(message)
-      length = Tokenizer.query(symbols[0..position]).size
+      length = (position > 0 ? Tokenizer.query(symbols[0..(position - 1)]).size + 1 : 0)
       raise "parsing query failed\n#{message}\n\n#{Tokenizer.query(symbols)}\n#{' ' * length}^"
     end
   end
+  # rubocop:enable Metrics/ClassLength
 end
 
 if $0 == __FILE__

data/lib/puppetdb_query/sync.rb

@@ -14,8 +14,8 @@ module PuppetDBQuery
       @destination = destination
     end
 
-    def sync(minutes = 5, seconds = 10)
-      logger.info "syncing puppetdb nodes and facts started"
+    def sync(minutes = 60, seconds = 10)
+      logger.info "syncing puppetdb nodes and facts started, running #{minutes} minutes"
 
       Timeout.timeout(60 * minutes - seconds) do
         updater = PuppetDBQuery::Updater.new(source, destination)
@@ -32,6 +32,9 @@ module PuppetDBQuery
             ts = Time.now
             updater.update3(timestamp - 2)
             timestamp = ts
+          rescue Timeout::Error
+            logger.info "syncing puppetdb nodes: now our time is up, we finsh"
+            return
           rescue
             logger.error $!
           end

data/lib/puppetdb_query/to_mongo.rb

@@ -18,7 +18,7 @@ module PuppetDBQuery
 
     private
 
-    # rubocop:disable Metrics/PerceivedComplexity
+    # rubocop:disable Metrics/PerceivedComplexity,Metrics/CyclomaticComplexity
     def query_term(term)
       # rubocop:disable Style/GuardClause
       if term.is_a?(Symbol)
@@ -33,26 +33,28 @@ module PuppetDBQuery
       # rubocop:enable Style/GuardClause
       terms = term.args.map { |t| query_term(t) }
       case term.operator.symbol
-      when :and
+      when :_and
         { :$and => terms }
-      when :or
+      when :_or
         { :$or => terms }
-      when :not
+      when :_not
         # $not currently (<=2.5.1) only supports negating equality operators.
         # so you can do { field: { $not : { [$eq,$gt,$lt,...] } }
         # but there is no way to negate an entire expression.
         # see https://jira.mongodb.org/browse/SERVER-10708
         { :$nor => terms }
-      when :equal
+      when :_equal
         { term.args[0] => term.args[1].to_s }
-      when :not_equal
+      when :_not_equal
         { term.args[0] => { :$ne => term.args[1].to_s } }
-      when :match
+      when :_match
         { term.args[0] => { :$regex => term.args[1].to_s } }
+      when :_in
+        { term.args[0] => { :$in => term.args[1] } }
       else
         raise "can't handle operator '#{term.operator}' yet"
       end
     end
-    # rubocop:enable Metrics/PerceivedComplexity
+    # rubocop:enable Metrics/PerceivedComplexity,Metrics/CyclomaticComplexity
   end
 end

data/lib/puppetdb_query/tokenizer.rb

@@ -2,39 +2,40 @@ require_relative "logging"
 
 module PuppetDBQuery
   # tokenize puppetdb queries
+  # FIXME: distinguish between language tokens and other tokens
   # rubocop:disable Metrics/ClassLength
   class Tokenizer
     include Logging
     include Enumerable
 
     SINGLE_CHAR_TO_TOKEN = {
-      "!" => :not,
-      "=" => :equal,
-      "(" => :begin,
-      ")" => :end,
-      "[" => :list_begin,
-      "]" => :list_end,
-      "<" => :less,
-      ">" => :greater,
-      "~" => :match,
-      "," => :comma,
+      "!" => :_not,
+      "=" => :_equal,
+      "(" => :_begin,
+      ")" => :_end,
+      "[" => :_list_begin,
+      "]" => :_list_end,
+      "<" => :_less,
+      ">" => :_greater,
+      "~" => :_match,
+      "," => :_comma,
     }.freeze
 
     DOUBLE_CHAR_TO_TOKEN = {
-      "!=" => :not_equal,
-      "!~" => :not_match,
-      "~>" => :match_array,
-      "<=" => :less_or_equal,
-      ">=" => :greater_or_equal,
+      "!=" => :_not_equal,
+      "!~" => :_not_match,
+      "~>" => :_match_array,
+      "<=" => :_less_or_equal,
+      ">=" => :_greater_or_equal,
     }.freeze
 
     STRING_TO_TOKEN = {
-      "not"   => :not,
-      "or"    => :or,
-      "and"   => :and,
-      "in"    => :in,
-      "is"    => :is,
-      "null"  => :null,
+      "not"   => :_not,
+      "or"    => :_or,
+      "and"   => :_and,
+      "in"    => :_in,
+      "is"    => :_is,
+      "null"  => :_null,
       "true"  => :true,
       "false" => :false,
     }.freeze
@@ -93,11 +94,13 @@ module PuppetDBQuery
       if DOUBLE_CHAR_TO_TOKEN.include?(s)
         increase
         increase
+        logger.debug "  resulting symbol: #{DOUBLE_CHAR_TO_TOKEN[s]}"
         return DOUBLE_CHAR_TO_TOKEN[s]
       end
       c = text[position]
       if SINGLE_CHAR_TO_TOKEN.include?(c)
         increase
+        logger.debug "  resulting symbol: #{SINGLE_CHAR_TO_TOKEN[c]}"
         return SINGLE_CHAR_TO_TOKEN[c]
       end
       case c
@@ -113,9 +116,9 @@ module PuppetDBQuery
     end
 
     def read_quoted
-      logger.debug "read quoted"
+      logger.debug "  read quoted"
       skip_whitespace
-      q = text[position]
+      q = text[position] # quote character
       increase
       r = ""
       while !empty? && (c = text[position]) != q
@@ -136,31 +139,33 @@ module PuppetDBQuery
       end
       error("I expected '#{q}' but I got '#{c}'") if c != q
       increase
-      logger.debug "resulting string: '#{r}'"
+      logger.debug "  resulting string: '#{r}'"
       r
     end
 
     def read_symbol
-      logger.debug "read symbol"
+      logger.debug "  read symbol"
       skip_whitespace
       r = ""
       while !empty? && (c = text[position]) =~ /[-a-zA-Z_0-9]/
         r << c
         increase
       end
-      logger.debug "resulting symbol: '#{r}'"
-      r.to_sym
+      s = STRING_TO_TOKEN[r]
+      s = r.to_sym unless s
+      logger.debug "  resulting symbol: #{s}"
+      s
     end
 
     def read_number
-      logger.debug "read number"
+      logger.debug "  read number"
       skip_whitespace
       r = ""
       while !empty? && (c = text[position]) =~ /[-0-9\.E]/
         r << c
         increase
       end
-      logger.debug "resulting number: '#{r}'"
+      logger.debug "  resulting number: '#{r}'"
       Integer(r)
     rescue
       Float(r)

data/lib/puppetdb_query/version.rb

@@ -1,3 +1,3 @@
 module PuppetDBQuery
-  VERSION = "0.0.8".freeze
+  VERSION = "0.0.9".freeze
 end

data/spec/puppetdb_query/parser_spec.rb

@@ -4,7 +4,7 @@ require "spec_helper"
 # rubocop:disable Style/MultilineMethodCallIndentation,Style/RedundantParentheses
 # rubocop:disable Style/ClosingParenthesisIndentation
 describe PuppetDBQuery::Parser do
-  PARSER_DATA = [
+  CORRECT_PARSER_DATA = [
     [ 'hostname=\'puppetdb-mike-217922\'',
       [PuppetDBQuery::Term.new(PuppetDBQuery::Parser::EQUAL).add(:hostname, "puppetdb-mike-217922")]
     ],
@@ -32,9 +32,29 @@ describe PuppetDBQuery::Parser do
     ],
   ].freeze
 
-  PARSER_DATA.each do |q, a|
+  CORRECT_PARSER_DATA.each do |q, a|
     it "translates correctly #{q.inspect}" do
       expect(subject.parse(q)).to eq(a)
     end
   end
+
+  it "complains about missing )" do
+    expect { subject.parse("a!=true and (b=false") }.to raise_error(/'\)' expected/)
+  end
+
+  it "complains about missing arguments for and" do
+    expect { subject.parse("a!=true and") }.to raise_error(/to few arguments for operator 'and'/)
+  end
+
+  it "complains about missing arguments for not" do
+    expect { subject.parse("not") }.to raise_error(/prefix operator 'not' got no argument/)
+  end
+
+  it "complains about closing bracket" do
+    expect { subject.parse(")") }.to raise_error(/that was not expected here: '\)'/)
+  end
+
+  it "complains about closing bracket 2" do
+    expect { subject.parse("not )") }.to raise_error(/that was not expected here: '\)'/)
+  end
 end

data/spec/puppetdb_query/to_mongo_spec.rb

@@ -27,7 +27,7 @@ describe PuppetDBQuery::ToMongo do
         ]
       }
     ],
-    [ '(group="develop-ci" or group=develop or group=mock) and (operatingsystemmajrelease="6")',
+    [ '(group="develop-ci" or group=develop or group=mock) and operatingsystemmajrelease="6"',
       { :$and => [
           { :$or => [
               { group: "develop-ci" },
@@ -39,10 +39,16 @@ describe PuppetDBQuery::ToMongo do
         ]
       }
     ],
-    [ "server_type=zoo or server_type='mesos-magr') and group!='infrastructure-ci'",
+    [ "server_type=zoo or server_type='mesos-magr' and group!='infrastructure-ci'",
       { :$or => [
           { server_type: "zoo" },
-          { server_type: "mesos-magr" }
+          { :$and => [
+               { server_type: "mesos-magr" },
+               { group:
+                   { :$ne => "infrastructure-ci" }
+               }
+            ]
+          }
         ]
       }
     ],
@@ -58,6 +64,11 @@ describe PuppetDBQuery::ToMongo do
         ]
       }
     ],
+    [ "server_type in [zoo, kafka]",
+      { server_type:
+          { :$in => [:zoo, :kafka] }
+      }
+    ],
   ].freeze
 
   TO_MONGO_DATA.each do |q, a|

data/spec/puppetdb_query/tokenizer_spec.rb

@@ -4,31 +4,32 @@ require "spec_helper"
 describe PuppetDBQuery::Tokenizer do
   TOKENIZER_DATA = [
     [ 'hostname=\'puppetdb-mike-217922\'',
-      [:hostname, :equal, "puppetdb-mike-217922"]
+      [:hostname, :_equal, "puppetdb-mike-217922"]
     ],
     [ 'disable_puppet = true',
-      [:disable_puppet, :equal, :true]
+      [:disable_puppet, :_equal, :true]
     ],
     [ 'fqdn~"app-dev" and group=develop and vertical~tracking and cluster_color~BLUE',
-      [:fqdn, :match, "app-dev", :and, :group, :equal, :develop, :and, :vertical, :match,
-       :tracking, :and, :cluster_color, :match, :BLUE]
+      [:fqdn, :_match, "app-dev", :_and, :group, :_equal, :develop, :_and, :vertical, :_match,
+       :tracking, :_and, :cluster_color, :_match, :BLUE]
     ],
     [ 'fqdn~"kafka" and group=develop and vertical=tracking',
-      [:fqdn, :match, "kafka", :and, :group, :equal, :develop, :and, :vertical, :equal, :tracking]
+      [:fqdn, :_match, "kafka", :_and, :group, :_equal, :develop, :_and, :vertical, :_equal,
+       :tracking]
     ],
     [ '(group="develop-ci" or group=develop or group=mock) and (operatingsystemmajrelease="6")',
-      [:begin, :group, :equal, "develop-ci", :or, :group, :equal, :develop, :or, :group, :equal,
-       :mock, :end, :and, :begin, :operatingsystemmajrelease, :equal, "6", :end]
+      [:_begin, :group, :_equal, "develop-ci", :_or, :group, :_equal, :develop, :_or, :group,
+       :_equal, :mock, :_end, :_and, :_begin, :operatingsystemmajrelease, :_equal, "6", :_end]
     ],
     [ "server_type=zoo or server_type='mesos-magr') and group!='infrastructure-ci'",
-      [:server_type, :equal, :zoo, :or, :server_type, :equal, "mesos-magr", :end, :and, :group,
-       :not_equal, "infrastructure-ci"]
+      [:server_type, :_equal, :zoo, :_or, :server_type, :_equal, "mesos-magr", :_end, :_and,
+       :group, :_not_equal, "infrastructure-ci"]
     ],
     [ "server_type~'mesos-magr' and group='ops-ci' and operatingsystemmajrelease=7 and" \
       " vmtest_vm!=true and disable_puppet!=true and puppet_artifact_version!=NO_VERSION_CHECK",
-      [:server_type, :match, "mesos-magr", :and, :group, :equal, "ops-ci", :and,
-       :operatingsystemmajrelease, :equal, 7, :and, :vmtest_vm, :not_equal, :true, :and,
-       :disable_puppet, :not_equal, :true, :and, :puppet_artifact_version, :not_equal,
+      [:server_type, :_match, "mesos-magr", :_and, :group, :_equal, "ops-ci", :_and,
+       :operatingsystemmajrelease, :_equal, 7, :_and, :vmtest_vm, :_not_equal, :true, :_and,
+       :disable_puppet, :_not_equal, :true, :_and, :puppet_artifact_version, :_not_equal,
        :NO_VERSION_CHECK]
     ],
   ].freeze

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: puppetdb_query
 version: !ruby/object:Gem::Version
-  version: 0.0.8
+  version: 0.0.9
 platform: ruby
 authors:
 - Michael Meyling
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-12-15 00:00:00.000000000 Z
+date: 2016-12-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake