puppetdb_query 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 483f7ff2212fc5b4b01df7c5e743848648e7796e
-  data.tar.gz: 86cecf8ac71a8ee6281f351c76a4ac10460def0e
+  metadata.gz: c33cfa1bbb82f1324f6716de9d17ed70308b35d2
+  data.tar.gz: 51b845298b9ded7a278ccf96481c96f2b287c936
 SHA512:
-  metadata.gz: 830b95936dd945d8540ee67a643d31faca45f4454b37eb2f2983809bed15d8e7594182fe29b84eee1e33977e30e97d45799dd0403752a146b32e61ce0c5321e7
-  data.tar.gz: 03b48691e946b2ecf0fc129e7a78a57c16f75c0d7fb267a843e8dc9b502d64cc80a5785a17089bae320c2b1d717171aa26058331c521c680a927b1c0d68b5eca
+  metadata.gz: 5f6c17658656b5452215c8f297bcd76ab4a48c1305e04b23e36e4fb9f1a504c5d0a9abe8256767b8295e8fb211a7d71c4ac5ccd35c8ae1b9acadb93c3f44cb35
+  data.tar.gz: 3eaeae64fda4338cf502e9bd2b4a304d24e6f975ff9c5310e74c0b91d6cc989d4fd5fee062f9a4df4ea6aa34a6bc2457a4b01b4c7a449121e8ba271d7eac2c3a

data/lib/puppetdb_query/mongodb.rb

@@ -8,7 +8,7 @@ module PuppetDBQuery
     include Logging
     attr_reader :connection
     attr_reader :nodes_collection
-    attr_reader :nodes_properties_collection
+    attr_reader :node_properties_collection
     attr_reader :meta_collection
 
     # initialize access to mongodb
@@ -18,12 +18,12 @@ module PuppetDBQuery
     #
     # @param connection        mongodb connection, should already be switched to correct database
     # @param nodes             symbol for collection that contains nodes with their facts
-    # @param nodes_properties  symbol for collection for nodes with their update timestamps
+    # @param node_properties   symbol for collection for nodes with their update timestamps
     # @param meta              symbol for collection with update metadata
-    def initialize(connection, nodes = :nodes, nodes_properties = :nodes_properties, meta = :meta)
+    def initialize(connection, nodes = :nodes, node_properties = :node_properties, meta = :meta)
       @connection = connection
       @nodes_collection = nodes
-      @nodes_propeties_collection = nodes_properties
+      @node_properties_collection = node_properties
       @meta_collection = meta
     end
 
@@ -89,18 +89,26 @@ module PuppetDBQuery
     end
 
     # update node properties
-    def node_properties_update(new_node_properties, ts_begin)
-      collection = connection[nodes_properties_collection]
+    def node_properties_update(new_node_properties)
+      collection = connection[node_properties_collection]
       old_names = collection.find.batch_size(999).projection(_id: 1).map { |k| k[:_id] }
       delete = old_names - new_node_properties.keys
-      collection.insert_many(nodes_properties.map { |k, v| v.dup.tap { v[:_id] = k } })
+      collection.insert_many(new_node_properties.map { |k, v| v.dup.tap { v[:_id] = k } })
       collection.delete_many(_id: { '$in' => delete })
-      ts_end = Time.iso8601(Time.now)
+    end
+
+    # update or insert timestamps for given fact update method
+    def meta_fact_update(method, ts_begin, ts_end)
       connection[meta_collection].find_one_and_update(
         {},
         {
           '$set' => {
-            last_node_properties_update: {
+            last_fact_update: {
+              ts_begin: ts_begin,
+              ts_end:   ts_end,
+              method:   method
+            },
+            method => {
               ts_begin: ts_begin,
               ts_end:   ts_end
             }
@@ -110,18 +118,13 @@ module PuppetDBQuery
       )
     end
 
-    # update or insert timestamps for given fact update method
-    def meta_fact_update(method, ts_begin, ts_end)
+    # update or insert timestamps for node_properties_update
+    def meta_node_properties_update(ts_begin, ts_end)
       connection[meta_collection].find_one_and_update(
         {},
         {
           '$set' => {
-            last_fact_update: {
-              ts_begin: ts_begin,
-              ts_end:   ts_end,
-              method:   method
-            },
-            method => {
+            last_node_properties_update: {
               ts_begin: ts_begin,
               ts_end:   ts_end
             }

data/lib/puppetdb_query/puppetdb.rb

@@ -24,8 +24,8 @@ module PuppetDBQuery
       api_nodes.map { |data| data['certname'] }
     end
 
-    # get array of node names
-    def nodes_properties
+    # get hash of node update properties
+    def node_properties
       result = {}
       api_nodes.each do |data|
         next if data['deactivated']
@@ -41,7 +41,7 @@ module PuppetDBQuery
     # get all nodes that have updated facts
     def nodes_update_facts_since(timestamp)
       ts = (timestamp.is_a?(String) ? Time.iso8601(ts) : timestamp)
-      nodes_properties.delete_if do |_k, data|
+      node_properties.delete_if do |_k, data|
         # TODO: in '/v3/nodes' we must take 'facts_timestamp'
         !data["facts-timestamp"] || Time.iso8601(data["facts-timestamp"]) < ts
       end.keys
@@ -74,7 +74,7 @@ module PuppetDBQuery
 
     def get_json(url, timeout)
       @lock.synchronize do
-        logger.info "get json from #{url}"
+        logger.info "  get json from #{url}"
         uri = URI.parse(url)
         http = Net::HTTP.new(uri.host, uri.port)
         http.use_ssl = uri.scheme == 'https'
@@ -82,7 +82,7 @@ module PuppetDBQuery
         request = Net::HTTP::Get.new(uri.request_uri)
         request['Accept'] = "application/json"
         response = http.request(request)
-        logger.info "  got #{response.body.size} characters from #{url}"
+        logger.info "    got #{response.body.size} characters from #{url}"
         JSON.parse(response.body)
       end
     end

data/lib/puppetdb_query/sync.rb

@@ -0,0 +1,48 @@
+require_relative 'logging'
+require_relative 'updater'
+
+module PuppetDBQuery
+  # sync node and fact data from source to destination
+  class Sync
+    include Logging
+
+    attr_reader :source
+    attr_reader :destination
+
+    def initialize(source, destination)
+      @source = source
+      @destination = destination
+    end
+
+    def sync(minutes = 5, seconds = 10)
+      logger.info "syncing puppetdb nodes and facts started"
+      Timeout.timeout(60 * minutes - seconds) do
+        updater = PuppetDBQuery::Updater.new(source, destination)
+
+        updater.update_node_properties
+
+        # make a full update
+        timestamp = Time.now
+        updater.update2
+
+        # make delta updates til our time is up
+        loop do
+          begin
+            ts = Time.now
+            updater.update3(timestamp - 2)
+            timestamp = ts
+          rescue
+            logger.error $!
+          end
+          logger.info "sleep for #{seconds} seconds"
+          sleep(seconds)
+        end
+      end
+      logger.info "syncing puppetdb nodes and facts ended"
+    rescue Timeout::Error
+      logger.info "syncing puppetdb nodes: now our time is up, we finsh"
+    rescue
+      logger.error $!
+    end
+  end
+end

data/lib/puppetdb_query/updater.rb

@@ -17,19 +17,24 @@ module PuppetDBQuery
     # update or insert facts for each one
     #
     # 335.6 seconds: update time for 1561 nodes
-    def update
+    def update1
+      logger.info "update1 started (full update)"
+      tsb = Time.now
       source_nodes = source.nodes
       destination_nodes = destination.nodes
-      (destination_nodes - source_nodes).each do |node|
-        destination.node_delete(node)
-      end
+      delete_missing(destination_nodes, source_nodes)
+      errors = false
       source_nodes.each do |node|
         begin
           destination.node_update(node, source.node_facts(node))
         rescue
+          errors = true
           logging.error $!
         end
       end
+      tse = Time.now
+      logger.info "update1 updated #{source_nodes.size} nodes in #{tse - tsb}"
+      destination.meta_fact_update("update1", tsb, tse) unless errors
     end
 
     # update by deleting missing nodes and get a complete map of nodes with facts
@@ -37,19 +42,24 @@ module PuppetDBQuery
     #
     # 166.4 seconds: update time for 1561 nodes
     def update2
+      logger.info "update2 started (full update)"
+      tsb = Time.now
       source_nodes = source.nodes
       destination_nodes = destination.nodes
-      (destination_nodes - source_nodes).each do |node|
-        destination.node_delete(node)
-      end
+      delete_missing(destination_nodes, source_nodes)
+      errors = false
       complete = source.facts
       complete.each do |node, facts|
         begin
           destination.node_update(node, facts)
         rescue
+          errors = true
           logging.error $!
         end
       end
+      tse = Time.now
+      logger.info "update2 updated #{source_nodes.size} nodes in #{tse - tsb}"
+      destination.meta_fact_update("update2", tsb, tse) unless errors
     end
 
     # update by deleting missing nodes and getting a list of nodes
@@ -57,20 +67,45 @@ module PuppetDBQuery
     #
     # update time depends extremly on the number of changed nodes
     def update3(last_update_timestamp)
+      logger.info "update3 started (incremental)"
+      tsb = Time.now
       source_nodes = source.nodes
       destination_nodes = destination.nodes
-      (destination_nodes - source_nodes).each do |node|
-        destination.node_delete(node)
-      end
+      delete_missing(destination_nodes, source_nodes)
+      errors = false
       modified = source.nodes_update_facts_since(last_update_timestamp)
       modified.each do |node|
         begin
           destination.node_update(node, source.node_facts(node))
         rescue
+          errors = true
           logging.error $!
         end
       end
-      modified.size
+      tse = Time.now
+      logger.info "update3 updated #{modified.size} nodes in #{tse - tsb}"
+      destination.meta_fact_update("update3", tsb, tse) unless errors
+    end
+
+    def update_node_properties
+      logger.info "update_node_properties started"
+      tsb = Time.now
+      source_node_properties = source.node_properties
+      destination.node_properties_update(source_node_properties)
+      tse = Time.now
+      logger.info "update_node_properties updated #{source_node_properties.size} nodes " \
+                  "in #{tse - tsb}"
+      destination.meta_node_properties_update(tsb, tse)
+    end
+
+    private
+
+    def delete_missing(destination_nodes, source_nodes)
+      missing = destination_nodes - source_nodes
+      missing.each do |node|
+        destination.node_delete(node)
+      end
+      logger.info "  deleted #{missing.size} nodes"
     end
   end
 end

data/lib/puppetdb_query/version.rb

@@ -1,3 +1,3 @@
 module PuppetDBQuery
-  VERSION = "0.0.3".freeze
+  VERSION = "0.0.4".freeze
 end

data/lib/puppetdb_query.rb

@@ -5,6 +5,7 @@ require_relative "puppetdb_query/tokenizer"
 require_relative "puppetdb_query/parser"
 require_relative "puppetdb_query/puppetdb"
 require_relative "puppetdb_query/mongodb"
+require_relative "puppetdb_query/sync"
 require_relative "puppetdb_query/to_mongo"
 require_relative "puppetdb_query/updater"
 require_relative "puppetdb_query/version"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: puppetdb_query
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - Michael Meyling
@@ -100,6 +100,7 @@ files:
 - lib/puppetdb_query/operator.rb
 - lib/puppetdb_query/parser.rb
 - lib/puppetdb_query/puppetdb.rb
+- lib/puppetdb_query/sync.rb
 - lib/puppetdb_query/term.rb
 - lib/puppetdb_query/to_mongo.rb
 - lib/puppetdb_query/tokenizer.rb