content_server 0.0.1 → 0.0.2

data/bin/backup_server

@@ -1,5 +1,10 @@
 #!/usr/bin/env ruby
 
+# Runs backup server. This server monitors a set of directories (blobs/patterns).
+# The files in those directories are indexed (calculating their SHA1).
+# A remote server copies new/changed files in the backup server. The backup server
+# index those new files too and send the content data (their SHA1 to the original server).
+
 begin
   require 'content_server'
 rescue LoadError

data/bin/content_server

@@ -1,5 +1,9 @@
 #!/usr/bin/env ruby
 
+# Runs content server. This server monitors a set of directories (blobs/patterns).
+# The files in those directories are indexed (calculating their SHA1).
+# Each unique content is backed up to the remote (backup) server.
+
 begin
   require 'content_server'
 rescue LoadError

data/lib/content_server/content_receiver.rb

@@ -1,39 +1,44 @@
-require 'eventmachine'
+require 'socket'
 
 module BBFS
   module ContentServer
 
     class ContentDataReceiver
-      def initialize queue, host, port
+      def initialize queue, port
         @queue = queue
-        @host = host
         @port = port
       end
 
-      def receive_data(data)
-        @queue.push(Marshal.load(data))
-      end
-
-      def start_server
-        EventMachine::start_server @host, @port, self
-        puts "Started ContentDataServer on #{@host}:#{@port}..."
+      def run
+        Socket.tcp_server_loop(@port) do |sock, client_addrinfo|
+          p 'Waiting on sock.gets.'
+          size_of_data = sock.read(4).unpack("l")[0]
+          p "Size of data: #{size_of_data}"
+          size_of_data = size_of_data.to_i
+          p "Size of data: #{size_of_data}"
+          data = sock.read(size_of_data)
+          p "Data received: #{data}"
+          unmarshaled_data = Marshal.load(data)
+          p "Unmarshaled data: #{unmarshaled_data}"
+          @queue.push unmarshaled_data
+        end
       end
     end
 
     class ContentDataSender
       def initialize host, port
-        @host = host
-        @port = port
+        p "Connecting to content server #{host}:#{port}."
+        @tcp_socket = TCPSocket.open(host, port)
       end
 
       def send_content_data content_data
-        send_data(Marshal.dump(content_data))
-      end
-
-      def connect
-        EventMachine.run {
-          EventMachine.connect @host, @port, self
-        }
+        p "Data to send: #{content_data}"
+        marshal_data = Marshal.dump(content_data)
+        p "Marshaled size: #{marshal_data.length}."
+        data_size = [marshal_data.length].pack("l")
+        p "Marshaled data: #{marshal_data}."
+        @tcp_socket.write data_size
+        @tcp_socket.write marshal_data
       end
     end
 

data/lib/content_server/queue_indexer.rb

@@ -1,3 +1,6 @@
+require 'file_indexing/index_agent'
+require 'file_indexing/indexer_patterns'
+
 module BBFS
   module ContentServer
 
@@ -13,22 +16,28 @@ module BBFS
 
       def run
         server_content_data = ContentData::ContentData.new
-        server_content_data.from_file(@content_data_path)
+        server_content_data.from_file(@content_data_path) rescue Errno::ENOENT
         # Start indexing on demand and write changes to queue
         thread = Thread.new do
           while true do
-            event = input_queue.pop
+            p 'Waiting on index input queue.'
+            event = @input_queue.pop
+            p "event: #{event}"
             # index files and add to copy queue
-            if (event[0] == FileStatEnum.CHANGED || event[0] == FileStatEnum.NEW)
-              index_agent = IndexAgent.new
-              indexer_patterns = IndexerPatterns.new
+            if event[0] == FileMonitoring::FileStatEnum::CHANGED || event[0] == FileMonitoring::FileStatEnum::NEW
+              p "Indexing content #{event[1]}."
+              index_agent = FileIndexing::IndexAgent.new
+              indexer_patterns = FileIndexing::IndexerPatterns.new
               indexer_patterns.add_pattern(event[1])
-              index_agent.index(indexer_patterns)
-              server_content_data.merge(index_agent.indexed_content)
-              # TODO(kolman) Don't write to file each change?
+              index_agent.index(indexer_patterns, server_content_data)
+              server_content_data.merge index_agent.indexed_content
+              # TODO(kolman): Don't write to file each change?
+              p "Writing server content data to #{@content_data_path}."
               server_content_data.to_file(@content_data_path)
-              output_queue.push(server_content_data)
+              p 'Adding server content data to queue.'
+              @output_queue.push(server_content_data)
             end
+            #p 'End of if.'
           end  # while true do
         end  # Thread.new do
         thread

data/lib/content_server.rb

@@ -1,8 +1,10 @@
 require 'content_data'
-require 'file_copy/copy'
+require 'file_copy'
+require 'file_indexing'
 require 'file_monitoring'
+require 'params'
+require 'set'
 require 'thread'
-require 'parameters'
 
 require_relative 'content_server/content_receiver'
 require_relative 'content_server/queue_indexer'
@@ -17,7 +19,7 @@ module BBFS
     PARAMS.parameter('remote_listening_port', 3333, 'Listening port for backup server content data.')
     PARAMS.parameter('backup_username', nil, 'Backup server username.')
     PARAMS.parameter('backup_password', nil, 'Backup server password.')
-    PARAMS.parameter('backup_destination_folder', File.expand_path('~/.bbfs/data'),
+    PARAMS.parameter('backup_destination_folder', File.expand_path('~/backup_data'),
                      'Backup server destination folder.')
     PARAMS.parameter('content_data_path', File.expand_path('~/.bbfs/var/content.data'),
                      'ContentData file path.')
@@ -44,18 +46,16 @@ module BBFS
       backup_server_content_data_queue = Queue.new
       content_data_receiver = ContentDataReceiver.new(
           backup_server_content_data_queue,
-          PARAMS.backup_server,
           PARAMS.remote_listening_port)
       # Start listening to backup server
       all_threads << Thread.new do
-        content_data_receiver.start_server
+        content_data_receiver.run
       end
 
       # # # # # # # # # # # # # #
       # Initialize/Start local indexer
-      copy_files_events = Queue.new
       local_server_content_data_queue = Queue.new
-      queue_indexer = QueueIndexer.new(copy_files_events,
+      queue_indexer = QueueIndexer.new(monitoring_events,
                                        local_server_content_data_queue,
                                        PARAMS.content_data_path)
       # Start indexing on demand and write changes to queue
@@ -63,27 +63,31 @@ module BBFS
 
       # # # # # # # # # # # # # # # # # # # # # #
       # Initialize/Start content data comparator
+      copy_files_events = Queue.new
       all_threads << Thread.new do
-        backup_server_content = nil
-        local_server_content = nil
+        backup_server_content_data = ContentData::ContentData.new
+        local_server_content_data = nil
         while true do
 
           # Note: This thread should be the only consumer of local_server_content_data_queue
-          # Note: The server will wait in the first time on pop until local sends it's content data
-          while !local_server_content || local_server_content_data_queue.size > 1
-            local_server_content_data = local_server_content_data_queue.pop
-          end
+          p 'Waiting on local server content data.'
+          local_server_content_data = local_server_content_data_queue.pop
 
           # Note: This thread should be the only consumer of backup_server_content_data_queue
           # Note: The server will wait in the first time on pop until backup sends it's content data
-          while !backup_server_content || backup_server_content_data_queue.size > 1
+          while backup_server_content_data_queue.size > 0
+            p 'Waiting on backup server content data.'
             backup_server_content_data = backup_server_content_data_queue.pop
           end
 
+          p 'Updating file copy queue.'
+          p "local_server_content_data #{local_server_content_data}."
+          p "backup_server_content_data #{backup_server_content_data}."
           # Remove backup content data from local server
-          content_to_copy = ContentData.remove(backup_server_content_data, local_server_content)
+          content_to_copy = ContentData::ContentData.remove(backup_server_content_data, local_server_content_data)
           # Add copy instruction in case content is not empty
-          output_queue.push(content_to_copy) unless content_to_copy.empty?
+          p "Content to copy: #{content_to_copy}"
+          copy_files_events.push(content_to_copy) unless content_to_copy.empty?
         end
       end
 
@@ -92,14 +96,19 @@ module BBFS
       # Start copying files on demand
       all_threads << Thread.new do
         while true do
+          p 'Waiting on copy files events.'
           copy_event = copy_files_events.pop
 
+          p "Copy file event: #{copy_event}"
+
           # Prepare source,dest map for copy.
           used_contents = Set.new
           files_map = Hash.new
-          copy_event.instances.each { |instance|
+          p "Instances: #{copy_event.instances}"
+          copy_event.instances.each { |key, instance|
+            p "Instance: #{instance}"
             # Add instance only if such content has not added yet.
-            if !used_contents.has_key?(instance.checksum)
+            if !used_contents.member?(instance.checksum)
               files_map[instance.full_path] = destination_filename(
                   PARAMS.backup_destination_folder,
                   instance.checksum)
@@ -107,14 +116,16 @@ module BBFS
             end
           }
 
+          p "Copying files: #{files_map}."
           # Copy files, waits until files are finished copying.
           FileCopy::sftp_copy(PARAMS.backup_username,
                               PARAMS.backup_password,
-                              PARAMS.backup_server,
+                              PARAMS.remote_server,
                               files_map)
         end
       end
 
+      all_threads.each { |t| t.abort_on_exception = true }
       all_threads.each { |t| t.join }
       # Should never reach this line.
     end
@@ -126,6 +137,7 @@ module BBFS
     def destination_filename(folder, sha1)
       File.join(folder, sha1[0,2], sha1[2,2], sha1[4,2], sha1)
     end
+    module_function :destination_filename
 
     def run_backup_server
       all_threads = []
@@ -143,9 +155,8 @@ module BBFS
 
       # # # # # # # # # # # # # #
       # Initialize/Start local indexer
-      copy_files_events = Queue.new
       local_server_content_data_queue = Queue.new
-      queue_indexer = QueueIndexer.new(copy_files_events,
+      queue_indexer = QueueIndexer.new(monitoring_events,
                                        local_server_content_data_queue,
                                        PARAMS.content_data_path)
       # Start indexing on demand and write changes to queue
@@ -158,12 +169,17 @@ module BBFS
           PARAMS.remote_listening_port)
       # Start sending to backup server
       all_threads << Thread.new do
-        content_data_sender.connect
         while true do
+          p 'Waiting on local server content data queue.'
           content_data_sender.send_content_data(local_server_content_data_queue.pop)
         end
       end
+
+      all_threads.each { |t| t.abort_on_exception = true }
+      all_threads.each { |t| t.join }
+      # Should never reach this line.
     end
+    module_function :run_backup_server
 
   end # module ContentServer
 end # module BBFS

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: content_server
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-04-08 00:00:00.000000000Z
+date: 2012-04-11 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: file_monitoring
-  requirement: &70305881380440 !ruby/object:Gem::Requirement
+  name: content_data
+  requirement: &70119917644580 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70305881380440
+  version_requirements: *70119917644580
 - !ruby/object:Gem::Dependency
-  name: file_indexing
-  requirement: &70305881379880 !ruby/object:Gem::Requirement
+  name: eventmachine
+  requirement: &70119917644120 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70305881379880
+  version_requirements: *70119917644120
 - !ruby/object:Gem::Dependency
-  name: parameters
-  requirement: &70305881379340 !ruby/object:Gem::Requirement
+  name: file_copy
+  requirement: &70119917643700 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70305881379340
+  version_requirements: *70119917643700
 - !ruby/object:Gem::Dependency
-  name: file_copy
-  requirement: &70305881378800 !ruby/object:Gem::Requirement
+  name: file_indexing
+  requirement: &70119917643240 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,21 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70305881378800
+  version_requirements: *70119917643240
 - !ruby/object:Gem::Dependency
-  name: content_data
-  requirement: &70305881378380 !ruby/object:Gem::Requirement
+  name: file_monitoring
+  requirement: &70119917642820 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70119917642820
+- !ruby/object:Gem::Dependency
+  name: params
+  requirement: &70119917574700 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,7 +76,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70305881378380
+  version_requirements: *70119917574700
 description: Monitor and Index a directory and back it up to backup server.
 email: kolmanv@gmail.com
 executables: