skydb 0.2.2 → 0.2.3

data/bin/sky

@@ -21,8 +21,10 @@ SkyDB.debug = true
 command :import do |c|
   c.syntax = 'sky import FILE'
   c.description = 'Imports data from a text file into a Sky table.'
+  c.option('--processes NUM', 'The number of processes to use.')
   c.option('--table STRING', 'The name of the table to import to.')
   c.option('--format STRING', 'The YAML format file to import with.')
+  c.option('--file-type STRING', 'The type of file being imported (tsv,json,csv,apache_log).')
   c.option('--headers STRING', 'A comma-delimited list of headers to use.')
   c.option('--append', 'Appends to an existing database if one exists.')
   c.option('--overwrite', 'Overwrites an existing database if one exists.')
@@ -39,6 +41,8 @@ command :import do |c|
     importer = SkyDB::Import::Importer.new()
     importer.table_name = options.table || ask("Table: ")
     importer.headers = options.headers.nil? ? nil : options.headers.split(/,/)
+    importer.file_type = options.file_type.nil? ? nil : options.file_type.to_sym
+    importer.processes = options.processes.nil? ? 1 : options.processes.to_i
     
     # Load transform files by name.
     formats = options.format || ask("Format: ")

data/lib/skydb.rb

@@ -2,6 +2,7 @@ require 'date'
 require 'msgpack'
 require 'socket'
 require 'treetop'
+require 'json'
 
 require 'skydb/action'
 require 'skydb/client'
@@ -40,12 +41,12 @@ class SkyDB
     :table_name, :table_name=,
     :multi, :ping, :lookup,
     :add_event,
-    :create_table, :delete_table, :get_table,
+    :create_table, :delete_table, :get_table, :get_tables,
     :add_action, :get_action, :get_actions,
     :add_property, :get_property, :get_properties,
     :next_actions,
     :aggregate,
-    :select
+    :query, :select
   ]
   
   

data/lib/skydb/action.rb

@@ -53,5 +53,24 @@ class SkyDB
     def to_msgpack
       return {id:id, name:name}.to_msgpack
     end
+
+    # Serializes the query object into a JSON string.
+    def to_json(*a); to_hash.to_json(*a); end
+
+    # Encodes the action into JSON format.
+    def to_hash(*a)
+      {
+        'id' => id,
+        'name' => name
+      }.delete_if {|k,v| v == '' || v == 0}
+    end
+
+    # Deserializes the selection field object from a hash.
+    def from_hash(hash, *a)
+      return nil if hash.nil?
+      self.id = hash['id'].to_i
+      self.name = hash['name']
+      return self
+    end
   end
 end

data/lib/skydb/client.rb

@@ -71,9 +71,14 @@ class SkyDB
     # Retrieves an individual table from the server, if it exists. Otherwise
     # returns nil.
     #
-    # @param [Fixnum] action_id  the identifier of the action to retrieve.
-    def get_table(action_id, options={})
-      return send_message(SkyDB::Message::GetTable.new(action_id, options))
+    # @param [Fixnum] name  the table name to retrieve.
+    def get_table(name, options={})
+      return send_message(SkyDB::Message::GetTable.new(name, options))
+    end
+
+    # Retrieves a list of all tables on the server.
+    def get_tables(options={})
+      return send_message(SkyDB::Message::GetTables.new(options))
     end
 
 
@@ -190,10 +195,15 @@ class SkyDB
     ####################################
 
     # Starts a query against the database.
+    def query()
+      return SkyDB::Query.new(:client => self)
+    end
+
+    # Starts a query with a single selection against the database.
     #
-    # @param [String] selection  a list of properties to select from the database.
+    # @param [String] fields  a list of properties to select from the database.
     def select(fields)
-      return SkyDB::Query.new(:client => self).select(fields)
+      return query.select(fields)
     end
 
 

data/lib/skydb/event.rb

@@ -25,12 +25,8 @@ class SkyDB
     # Object ID
     ##################################
 
-    # The numeric identifier of the object that the event is attached to.
-    attr_reader :object_id
-    
-    def object_id=(value)
-      @object_id = value.to_i
-    end
+    # The object identifier.
+    attr_accessor :object_id
 
     ##################################
     # Timestamp
@@ -103,7 +99,7 @@ class SkyDB
     # Encodes the event into MsgPack format.
     def to_msgpack
       obj = {
-        :objectId => object_id,
+        :objectId => object_id.to_msgpack,
         :timestamp => SkyDB::Timestamp.to_timestamp(timestamp)
       }
       obj[:action] = action unless action.nil? || action.empty?

data/lib/skydb/import/importer.rb

@@ -1,6 +1,12 @@
 require 'yaml'
 require 'csv'
+require 'yajl'
+require 'zlib'
+require 'bzip2'
+require 'open-uri'
 require 'ruby-progressbar'
+require 'apachelogregex'
+require 'useragent'
 
 class SkyDB
   class Import
@@ -11,6 +17,7 @@ class SkyDB
       #
       ##########################################################################
 
+      class UnsupportedFileType < StandardError; end
       class TransformNotFound < StandardError; end
       
 
@@ -28,6 +35,7 @@ class SkyDB
         self.table_name  = options[:table_name]
         self.format = options[:format]
         self.files  = options[:files] || []
+        self.processes = options[:processes] || 1
       end
     
 
@@ -37,6 +45,9 @@ class SkyDB
       #
       ##########################################################################
 
+      # The number of processes to use.
+      attr_accessor :processes
+      
       # The client to access the Sky server with.
       attr_accessor :client
 
@@ -56,6 +67,10 @@ class SkyDB
       # treat the CSV input as not having a header row.
       attr_accessor :headers
 
+      # The file type of file being imported can be one of
+      # :csv, :tsv, :json, :apache_log
+      attr_accessor :file_type
+
 
       ##########################################################################
       #
@@ -67,83 +82,244 @@ class SkyDB
       # Import
       ##################################
     
-      # Imports the rows from a list of files.
+      # Imports records from a list of files.
       #
       # @param [Array]  a list of files to import.
-      def import(files)
+      def import(files, options={})
         files = [files] unless files.is_a?(Array)
+        options[:progress_bar] = true unless options.has_key?(:progress_bar)
+        progress_bar = nil
         
         # Set the table to import into.
         SkyDB.table_name = table_name
-        
-        # Loop over each of the files.
-        files.each do |file|
-          # Initialize progress bar.
-          count = %x{wc -l #{file}}.split.first.to_i
-          progress_bar = ::ProgressBar.create(
-            :total => count,
-            :format => ('%-40s' % file) + ' |%B| %P%%'
-          )
 
-          # Determine column separator by extension.
-          col_sep = ','
-          if File.extname(file) == '.tsv' || File.extname(file) == '.txt'
-            col_sep = "\t"
+        # Initialize progress bar.
+        count = files.inject(0) do |cnt,file|
+          # disable progress bar if using compressed files
+          if Dir.glob(file).detect{|f|['.gz','.bz2'].include?(File.extname(f).downcase)}
+            options[:progress_bar] = false
+            break
           end
+          cnt + %x{wc -l #{file}|tail -1}.split.first.to_i
+        end
+        progress_bar = ::ProgressBar.create(:total => count, :format => '|%B| %P%%') if (options[:progress_bar] and self.processes == 1)
 
-          file = File.open(file, 'r')
-          begin
+        # Loop over each of the files.
+        files_expanded = files.inject([]) {|fs,fg| fs.concat(Dir[File.expand_path(fg)].delete_if{|f| File.directory?(f)}); fs}
+        file_groups =
+          if processes > 1
+            files_per_group = (files_expanded.size/Float(self.processes)).ceil
+            files_expanded.each_slice(files_per_group).to_a
+          else
+            [files_expanded]
+          end
+        process_ids = []
+  
+        for i in (0...processes)
+          process_ids << fork do
             SkyDB.multi(:max_count => 1000) do
-              # Process each line of the CSV file.
-              CSV.foreach(file, :headers => headers.nil?, :col_sep => col_sep) do |row|
-                input = {}
-                
-                # If headers were not specified then use the ones from the
-                # CSV file and just convert the row to a hash.
-                if headers.nil?
-                  input = row.to_hash
-                
-                # If headers were specified then manually convert the row
-                # using the headers provided.
-                else
-                  headers.each_with_index do |header, index|
-                    input[header] = row[index]
+              file_groups[i].each do |file|
+                # puts "process[#{i}] -> #{file}"
+                each_record(file, options) do |input|
+                  # Convert input line to a symbolized hash.
+                  output = translate(input)
+                  output._symbolize_keys!
+            
+                  # p output
+ 
+                  if output[:object_id].nil?
+                    progress_bar.clear() unless progress_bar.nil?
+                    $stderr.puts "[ERROR] Object id required on line #{$.}"
+                  elsif output[:timestamp].nil?
+                    progress_bar.clear() unless progress_bar.nil?
+                    $stderr.puts "[ERROR] Invalid timestamp on line #{$.}"
+                  else
+                    # Convert hash to an event and send to Sky.
+                    event = SkyDB::Event.new(output)
+                    SkyDB.add_event(event)
                   end
+          
+                  # Update progress bar.
+                  progress_bar.increment() unless progress_bar.nil?
                 end
-                
-                # Convert input line to a symbolized hash.
-                output = translate(input)
-                output._symbolize_keys!
-                
-                # p output
-                
-                # Convert hash to an event and send to Sky.
-                event = SkyDB::Event.new(output)
-
-                if !(event.object_id > 0)
-                  progress_bar.clear()
-                  puts "[ERROR] Invalid object id on line #{$.}."
-                elsif event.timestamp.nil?
-                  progress_bar.clear()
-                  puts "[ERROR] Invalid timestamp on line #{$.}."
-                else
-                  SkyDB.add_event(event)
-                end
-              
-                # Update progress bar.
-                progress_bar.increment()
               end
             end
-          ensure
-            file.close
           end
+        end
+        process_ids.each { |process_id|  Process.waitpid(process_id) }
+
+        # Finish progress bar.
+        progress_bar.finish() unless progress_bar.nil? || progress_bar.finished?
+        
+        return nil
+      end
+
 
-          # Finish progress bar.
-          progress_bar.finish()        
+      ##################################
+      # File Iteration
+      ##################################
+    
+      def file_foreach(file, &block)
+        case File.extname(file).downcase
+        when '.bz2'
+          Bzip2::Reader.foreach(file) do |line|
+            yield line
+          end
+        when '.gz'
+          Zlib::GzipReader.open(file) do |f|
+            f.each_line(file) do |line|
+              yield line
+            end
+          end
+        else
+          File.foreach(file) do |line|
+            yield line
+          end
+        end
+      end
+
+
+      ##################################
+      # Iteration
+      ##################################
+    
+      # Executes a block for each record in a given file. A record is defined
+      # by the file's type (:csv, :tsv, :json).
+      #
+      # @param [String] file  the path to the file to iterate over.
+      def each_record(file, options)
+        # Determine file type automatically if not passed in.
+        if self.file_type.nil?
+          self.file_type = 
+            case File.extname(file)
+            when '.tsv' then :tsv
+            when '.txt' then :tsv
+            when '.json' then :json
+            when '.csv' then :csv
+            when '.log' then :apache_log
+            end
+          warn("[import] Determining file type: #{self.file_type || '???'}")
+        end
+        
+        # Process the record by file type.
+        case self.file_type
+        when :csv then each_text_record(file, ",", options, &Proc.new)
+        when :tsv then each_text_record(file, "\t", options, &Proc.new)
+        when :json then each_json_record(file, options, &Proc.new)
+        when :apache_log then each_apache_log_record(file, options, &Proc.new)
+        else raise SkyDB::Import::Importer::UnsupportedFileType.new("File type not supported by importer: #{file_type || File.extname(file)}")
         end
         
         return nil
       end
+      
+      # Executes a block for each line of a delimited flat file format
+      # (CSV, TSV).
+      #
+      # @param [String] file  the path to the file to iterate over.
+      # @param [String] col_sep  the column separator.
+      def each_text_record(file, col_sep, options)
+        # Process each line of the CSV file.
+        CSV.foreach(file, :headers => headers.nil?, :col_sep => col_sep) do |row|
+          record = nil
+          
+          # If headers were not specified then use the ones from the
+          # CSV file and just convert the row to a hash.
+          if headers.nil?
+            record = row.to_hash
+          
+          # If headers were specified then manually convert the row
+          # using the headers provided.
+          else
+            record = {}
+            headers.each_with_index do |header, index|
+              record[header] = row[index]
+            end
+          end
+
+          # Skip over blank rows.
+          next if record.values.reject{|v| v == '' || v.nil? }.length == 0
+
+          yield(record)
+        end
+      end
+
+      # Executes a block for each line of a JSON file.
+      #
+      # @param [String] file  the path to the file to iterate over.
+      def each_json_record(file, options)
+        io = open(file)
+
+        # Process each line of the JSON file.
+        Yajl::Parser.parse(io) do |record|
+          yield(record)
+        end
+      end
+
+      # Executes a block for each line of a standard Apache log file.
+      #
+      # @param [String] file  the path to the file to iterate over.
+      def each_apache_log_record(file, options)
+        format = options[:format] || '%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"'
+        parser = ApacheLogRegex.new(format)
+
+        file_foreach(file) do |line|
+          begin
+            hash = parser.parse!(line)
+            m, method, url = *hash['%r'].to_s.match(/^(\w+) ([^ ]+)/)
+            uri = URI.parse("http://localhost#{path}") rescue nil
+            record = {
+              :ip_address => hash['%h'],
+              :timestamp => DateTime.strptime(hash['%t'].gsub(/\[|\]/, ''), "%d/%b/%Y:%H:%M:%S %z"),
+              :method => method,
+              :url => url,
+              :status_code => hash['%s'],
+              :size => hash['%b'],
+            }
+            record[:user_identifier] = hash['%l'] unless hash['%l'] == '-'
+            record[:user_id] = hash['%u'] unless hash['%u'] == '-'
+
+            # Extract the parts of the URI.
+            if !uri.nil?
+              record[:path] = uri.path
+              record[:query_string] = uri.query
+              record[:query] = CGI::parse(uri.query) rescue {}
+              record[:fragment] = uri.fragment
+            end
+
+            # Extract the referrer if there is one.
+            if !hash['%{Referer}i'].nil? && hash['%{Referer}i'] != '-'
+              record[:referer] = hash['%{Referer}i']
+              referer_uri = URI.parse(record[:referer]) rescue nil
+              if !referer_uri.nil?
+                record[:referer_host] = referer_uri.host
+                record[:referer_path] = referer_uri.path
+                record[:referer_query_string] = referer_uri.query
+                record[:referer_query] = CGI::parse(referer_uri.query) rescue {}
+              end
+            end
+            
+            # Extract specific user agent information.
+            if !hash['%{User-Agent}i'].nil?
+              user_agent = UserAgent.parse(hash['%{User-Agent}i'])
+              record[:user_agent] = hash['%{User-Agent}i']
+              record[:ua_name] = user_agent.browser.to_s unless user_agent.browser.nil?
+              record[:ua_version] = user_agent.version.to_s unless user_agent.version.nil?
+              record[:ua_platform] = user_agent.platform.to_s unless user_agent.platform.nil?
+              record[:ua_os] = user_agent.os.to_s unless user_agent.os.nil?
+              record[:ua_mobile] = user_agent.mobile?
+            end
+
+            # Skip junk log entries.
+            next if method == "HEAD" || method == "OPTIONS"
+
+            yield(record)
+
+          rescue ApacheLogRegex::ParseError => e
+            $stderr.puts "[ERROR] Unable to parse line #{$.} in #{file} (#{e.message})"
+          end
+        end
+      end
 
 
       ##################################
@@ -156,12 +332,14 @@ class SkyDB
       #
       # @return [Hash]  the output hash.
       def translate(input)
-        output = {}
+        output = {:action => {}, :data => {}}
 
         translators.each do |translator|
           translator.translate(input, output)
         end
 
+        output.delete(:action) if output[:action].keys.length == 0
+        output.delete(:data) if output[:data].keys.length == 0
         return output
       end
       
@@ -202,7 +380,6 @@ class SkyDB
       # @param [Hash]  the hash of transform info.
       # @param [Array]  the path of fields.
       def load_transform_fields(fields, path=nil)
-        
         # Convert each field to a translator.
         fields.each_pair do |key, value|
           translator = Translator.new(:output_field => (path.nil? ? key : path.clone.concat([key])))