RubyGems - content_server - Versions diffs - 0.0.10 → 1.0.0 - Mend

content_server 0.0.10 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (14) hide show

data/bin/backup_server +26 -14
data/bin/content_server +26 -24
data/lib/content_server.rb +10 -186
data/lib/content_server/backup_server.rb +120 -0
data/lib/content_server/content_receiver.rb +44 -45
data/lib/content_server/content_server.rb +89 -0
data/lib/content_server/file_streamer.rb +225 -220
data/lib/content_server/queue_copy.rb +190 -182
data/lib/content_server/queue_indexer.rb +91 -92
data/lib/content_server/remote_content.rb +70 -72
data/lib/content_server/version.rb +1 -3
data/spec/content_server/content_server_spec.rb +10 -12
data/spec/content_server/file_streamer_spec.rb +50 -52
metadata +246 -33

data/lib/content_server/remote_content.rb CHANGED

@@ -5,93 +5,91 @@ require 'log'
 require 'networking/tcp'
 require 'params'
-module BBFS
-  module ContentServer
+module ContentServer
+  Params.integer('remote_content_fetch_timeout', 10, 'Remote content desired freshness in seconds.')
+  Params.integer('remote_content_save_timeout', 60*60, 'Remote content force refresh in seconds.')
+  # TODO(kolman): Use only one tcp/ip socket by utilizing one NQueue for many queues!
+  class RemoteContentClient
+    def initialize(dynamic_content_data, host, port, local_backup_folder)
+      @dynamic_content_data = dynamic_content_data
+      @remote_tcp = Networking::TCPClient.new(host, port, method(:receive_content))
+      @last_fetch_timestamp = nil
+      @last_save_timestamp = nil
+      @content_server_content_data_path = File.join(local_backup_folder, 'remote',
+                                                    host + '_' + port.to_s)
+      Log.debug2("Initialized RemoteContentClient: host:#{host}   port:#{port}  local_backup_folder:#{local_backup_folder}")
+    end
-    Params.integer('remote_content_timeout', 10, 'Remote content desired freshness in seconds.')
-    Params.integer('max_content_timeout', 60*60, 'Remote content force refresh in seconds.')
+    def receive_content(message)
+      Log.debug1("Backup server received Remote content data:#{message.to_s}")
+      ref = @dynamic_content_data.last_content_data
+      @dynamic_content_data.update(message)
+      @last_fetch_timestamp = Time.now.to_i
-    # TODO(kolman): Use only one tcp/ip socket by utilizing one NQueue for many queues!
-    class RemoteContent
-      def initialize(dynamic_content_data, host, port, local_backup_folder)
-        @dynamic_content_data = dynamic_content_data
-        @remote_tcp = Networking::TCPClient.new(host, port, method(:receive_content))
-        @last_update_timestamp = nil
-        @content_server_content_data_path = File.join(local_backup_folder, 'remote',
-                                                      host + '_' + port.to_s)
+      save_time_span = Params['remote_content_save_timeout']
+      if !@last_save_timestamp.nil?
+        save_time_span = Time.now.to_i - @last_save_timestamp
       end
-      def receive_content(message)
-        Log.debug1("Remote content data received: #{message.to_s}")
-        ref = @dynamic_content_data.last_content_data
-        @dynamic_content_data.update(message)
-        max_time_span = Params['max_content_timeout']
-        if !@last_update_timestamp.nil?
-          max_time_span = Time.now.to_i - @last_update_timestamp
-        end
-        @last_update_timestamp = Time.now.to_i
-        if ref != message || max_time_span >= Params['max_content_timeout']
-          Log.debug2("Remote content data changed or max time span is large, writing.")
-          Log.debug3("max_time_span: #{max_time_span}")
-          write_to = File.join(@content_server_content_data_path,
-                               @last_update_timestamp.to_s + '.cd')
-          FileUtils.makedirs(@content_server_content_data_path) unless \
+      if save_time_span >= Params['remote_content_save_timeout']
+      	@last_save_timestamp = Time.now.to_i
+        write_to = File.join(@content_server_content_data_path,
+                             @last_save_timestamp.to_s + '.cd')
+        FileUtils.makedirs(@content_server_content_data_path) unless \
               File.directory?(@content_server_content_data_path)
-          count = File.open(write_to, 'wb') { |f| f.write(message.to_s) }
-        else
-          Log.debug2("No need to write remote content data, it has not changed.")
-        end
+        count = File.open(write_to, 'wb') { |f| f.write(message.to_s) }
+        Log.info("Written content data to file:#{write_to}.")
+      else
+        Log.info("No need to write remote content data, it has not changed.")
       end
+    end
-      def run()
-        threads = []
-        threads << @remote_tcp.tcp_thread if @remote_tcp != nil
-        threads << Thread.new do
-          loop do
-            # if need content data
-            if @last_update_timestamp.nil?
-              sleep_time_span = Params['remote_content_timeout']
-            else
-              sleep_time_span = Time.now.to_i - @last_update_timestamp
-            end
-            if sleep_time_span >= Params['remote_content_timeout']
-              # Send ping!
-              Log.debug2('Sending remote contend request.')
-              bytes_written = @remote_tcp.send_obj(nil)
-              Log.debug3("Bytes written #{bytes_written}.")
-            end
-            sleep_time_span = Time.now.to_i - @last_update_timestamp \
-                unless @last_update_timestamp.nil?
-            Log.debug2("sleep_time_span: #{sleep_time_span}")
-            sleep(sleep_time_span) if sleep_time_span > 0
+    def run()
+      Log.debug1("Running remote content client.")
+      threads = []
+      threads << @remote_tcp.tcp_thread if @remote_tcp != nil
+      threads << Thread.new do
+        Log.debug1("New thread.")
+        loop do
+          # if need content data
+          sleep_time_span = Params['remote_content_save_timeout']
+          if !@last_fetch_timestamp.nil?
+            sleep_time_span = Time.now.to_i - @last_fetch_timestamp
+          end
+          Log.debug1("sleep_time_span: #{sleep_time_span}")
+          if sleep_time_span >= Params['remote_content_save_timeout']
+            # Send ping!
+            bytes_written = @remote_tcp.send_obj(nil)
+            Log.info("sending ping request for remote content data!")
           end
+          sleep(sleep_time_span) if sleep_time_span > 0
         end
       end
     end
+  end
-    class RemoteContentClient
-      def initialize(dynamic_content_data, port)
-        @dynamic_content_data = dynamic_content_data
-        @tcp_server = Networking::TCPServer.new(port, method(:content_requested))
-      end
-      def content_requested(addr_info, message)
-        # Send response.
-        Log.debug1('Local content data requested.')
-        @tcp_server.send_obj(@dynamic_content_data.last_content_data)
-      end
+  class RemoteContentServer
+    def initialize(dynamic_content_data, port)
+      @dynamic_content_data = dynamic_content_data
+      @tcp_server = Networking::TCPServer.new(port, method(:content_requested))
+      Log.debug2("initialize RemoteContentServer on port:#{port}")
+    end
-      def tcp_thread
-        return @tcp_server.tcp_thread if @tcp_server != nil
-        nil
-      end
+    def content_requested(addr_info, message)
+      # Send response.
+      Log.info("Master server received content data request.  Sending content data:#{@dynamic_content_data.last_content_data}")
+      @tcp_server.send_obj(@dynamic_content_data.last_content_data)
+      Log.info('Master server sent content data')
+    end
+    def tcp_thread
+      return @tcp_server.tcp_thread if @tcp_server != nil
+      nil
     end
   end
 end

data/lib/content_server/version.rb CHANGED

@@ -1,5 +1,3 @@
-module BBFS
   module ContentServer
-    VERSION = "0.0.10"
+    VERSION = "1.0.0"
   end
-end

data/spec/content_server/content_server_spec.rb CHANGED

@@ -2,26 +2,24 @@ require 'rspec'
 require_relative '../../lib/file_copy/copy.rb'
-module BBFS
-  module ContentServer
-    module Spec
+module ContentServer
+  module Spec
-      describe 'Backup Listener' do
+    describe 'Backup Listener' do
-      end
-      describe 'Local file monitor' do
+    end
-      end
+    describe 'Local file monitor' do
-      describe 'Local file indexer' do
+    end
-      end
+    describe 'Local file indexer' do
-      describe 'File copier' do
+    end
-      end
+    describe 'File copier' do
     end
   end
 end

data/spec/content_server/file_streamer_spec.rb CHANGED

@@ -5,66 +5,64 @@ require 'stringio'
 require_relative '../../lib/content_server/file_streamer'
 # Uncomment to debug spec.
-BBFS::Params['log_write_to_console'] = false
-BBFS::Params['log_write_to_file'] = false
-BBFS::Params['log_debug_level'] = 0
-BBFS::Params['streaming_chunk_size'] = 5
-BBFS::Params.init ARGV
-BBFS::Log.init
+Params['log_write_to_console'] = false
+Params['log_write_to_file'] = false
+Params['log_debug_level'] = 0
+Params['streaming_chunk_size'] = 5
+Params.init ARGV
+Log.init
 # Have to be set to test chunking mechanism.
-module BBFS
-  module ContentServer
-    module Spec
-      describe 'FileStreamer' do
-        it 'should copy one file chunk by chunks and validate content' do
-          Log.info('#0 start')
-          orig_file = StringIO.new('Some content. Some content. Some content. Some content.')
-          Log.info("orig_file #{orig_file.to_s}.")
+module ContentServer
+  module Spec
+    describe 'FileStreamer' do
+      it 'should copy one file chunk by chunks and validate content' do
+        Log.info('#0 start')
+        orig_file = StringIO.new('Some content. Some content. Some content. Some content.')
+        Log.info("orig_file #{orig_file.to_s}.")
-          # should simulate Tempfile object, thus need to add to this object Tempfile methsods
-          # that are absent in StringIO, but used in tested ruby code
-          dest_file = StringIO.new
-          def dest_file.path
-            '/tmp/path/tmp_basename'
-          end
-          def dest_file.unlink
-            true
-          end
-          Log.info("dest_file #{dest_file.to_s}.")
+        # should simulate Tempfile object, thus need to add to this object Tempfile methsods
+        # that are absent in StringIO, but used in tested ruby code
+        dest_file = StringIO.new
+        def dest_file.path
+          '/tmp/path/tmp_basename'
+        end
+        def dest_file.unlink
+          true
+        end
+        Log.info("dest_file #{dest_file.to_s}.")
-          streamer = nil
-          done = lambda{ |checksum, filename|
-            Log.info('#4 streaming done, check content ok.')
-            dest_file.string().should eq(orig_file.string())
+        streamer = nil
+        done = lambda{ |checksum, filename|
+          Log.info('#4 streaming done, check content ok.')
+          dest_file.string().should eq(orig_file.string())
-            Log.info('#5 exiting streamer thread.')
-            streamer.thread.exit
-          }
+          Log.info('#5 exiting streamer thread.')
+          streamer.thread.exit
+        }
-          receiver = BBFS::ContentServer::FileReceiver.new(done)
-          send_chunk = lambda { |*args|
-            receiver.receive_chunk(*args)
-            streamer.copy_another_chuck('da39a3ee5e6b4b0d3255bfef95601890afd80709')
-          }
+        receiver = ContentServer::FileReceiver.new(done)
+        send_chunk = lambda { |*args|
+          receiver.receive_chunk(*args)
+          streamer.copy_another_chuck('da39a3ee5e6b4b0d3255bfef95601890afd80709')
+        }
-          Log.info('#2 start streaming.')
-          # This is for FileStreamer :NEW_STREAM and FileReceiver :receive_chunk
-          ::File.stub(:new).and_return(orig_file, dest_file)
-          ::FileUtils.stub(:makedirs).and_return(true)
-          ::FileUtils.stub(:copy_file).and_return(true)
-          # This is for FileReceiver :handle_last_chunk
-          ::File.stub(:rename)
-          # This is for Index agent 'get_checksum' which opens file, read content and validates
-          # checksum.
-          ::File.stub(:open).and_return(dest_file)
+        Log.info('#2 start streaming.')
+        # This is for FileStreamer :NEW_STREAM and FileReceiver :receive_chunk
+        ::File.stub(:new).and_return(orig_file, dest_file)
+        ::FileUtils.stub(:makedirs).and_return(true)
+        ::FileUtils.stub(:copy_file).and_return(true)
+        # This is for FileReceiver :handle_last_chunk
+        ::File.stub(:rename)
+        # This is for Index agent 'get_checksum' which opens file, read content and validates
+        # checksum.
+        ::File.stub(:open).and_return(dest_file)
-          streamer = BBFS::ContentServer::FileStreamer.new(send_chunk)
-          Log.info('#3 start streaming.')
-          streamer.start_streaming('da39a3ee5e6b4b0d3255bfef95601890afd80709', 'dummy')
-          streamer.thread.join()
-          sleep Params['log_param_max_elapsed_time_in_seconds_from_last_flush'] + 1
-        end
+        streamer = ContentServer::FileStreamer.new(send_chunk)
+        Log.info('#3 start streaming.')
+        streamer.start_streaming('da39a3ee5e6b4b0d3255bfef95601890afd80709', 'dummy')
+        streamer.thread.join()
+        Log.flush
       end
     end
   end

metadata CHANGED

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: content_server
 version: !ruby/object:Gem::Version
-  version: 0.0.10
+  version: 1.0.0
   prerelease:
 platform: ruby
 authors:
@@ -9,87 +9,298 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2012-12-02 00:00:00.000000000Z
+date: 2013-05-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: content_data
-  requirement: &16909160 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.0.0
   type: :runtime
   prerelease: false
-  version_requirements: *16909160
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
 - !ruby/object:Gem::Dependency
   name: file_indexing
-  requirement: &16908640 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.0.0
   type: :runtime
   prerelease: false
-  version_requirements: *16908640
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
 - !ruby/object:Gem::Dependency
   name: file_monitoring
-  requirement: &16908120 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.0.0
   type: :runtime
   prerelease: false
-  version_requirements: *16908120
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
 - !ruby/object:Gem::Dependency
   name: log
-  requirement: &16907640 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.0.0
   type: :runtime
   prerelease: false
-  version_requirements: *16907640
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
 - !ruby/object:Gem::Dependency
   name: networking
-  requirement: &16907220 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.0.0
   type: :runtime
   prerelease: false
-  version_requirements: *16907220
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
 - !ruby/object:Gem::Dependency
   name: params
-  requirement: &16906580 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+- !ruby/object:Gem::Dependency
+  name: process_monitoring
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.9.2.2
   type: :runtime
   prerelease: false
-  version_requirements: *16906580
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.9.2.2
 - !ruby/object:Gem::Dependency
   name: run_in_background
-  requirement: &16905880 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+- !ruby/object:Gem::Dependency
+  name: content_data
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+- !ruby/object:Gem::Dependency
+  name: file_indexing
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.0.0
   type: :runtime
   prerelease: false
-  version_requirements: *16905880
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+- !ruby/object:Gem::Dependency
+  name: file_monitoring
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+- !ruby/object:Gem::Dependency
+  name: log
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+- !ruby/object:Gem::Dependency
+  name: networking
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+- !ruby/object:Gem::Dependency
+  name: params
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+- !ruby/object:Gem::Dependency
+  name: process_monitoring
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.9.2.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.9.2.2
+- !ruby/object:Gem::Dependency
+  name: run_in_background
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
 description: Monitor and Index a directory and back it up to backup server.
-email: kolmanv@gmail.com
+email: bbfsdev@gmail.com
 executables:
 - content_server
 - backup_server
@@ -99,15 +310,17 @@ files:
 - lib/content_server.rb
 - lib/content_server/file_streamer.rb
 - lib/content_server/queue_copy.rb
+- lib/content_server/content_server.rb
 - lib/content_server/content_receiver.rb
 - lib/content_server/remote_content.rb
 - lib/content_server/version.rb
+- lib/content_server/backup_server.rb
 - lib/content_server/queue_indexer.rb
 - spec/content_server/content_server_spec.rb
 - spec/content_server/file_streamer_spec.rb
 - bin/content_server
 - bin/backup_server
-homepage: http://github.com/kolmanv/bbfs
+homepage: http://github.com/bbfsdev/bbfs
 licenses: []
 post_install_message:
 rdoc_options: []
@@ -127,7 +340,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 1.8.10
+rubygems_version: 1.8.24
 signing_key:
 specification_version: 3
 summary: Servers for backing up content.