skydb 0.2.1 → 0.2.2

data/bin/sky

@@ -0,0 +1,85 @@
+#!/usr/bin/env ruby
+
+$:.unshift(File.join(File.dirname(File.expand_path(__FILE__)), '..', 'lib'))
+
+require 'rubygems'
+require 'skydb'
+require 'skydb/import'
+require 'commander/import'
+
+program :name, 'Sky'
+program :version, SkyDB::VERSION
+program :description, 'A multi-purpose utilty for the Sky database.'
+
+SkyDB.debug = true
+
+
+################################################################################
+# Import
+################################################################################
+
+command :import do |c|
+  c.syntax = 'sky import FILE'
+  c.description = 'Imports data from a text file into a Sky table.'
+  c.option('--table STRING', 'The name of the table to import to.')
+  c.option('--format STRING', 'The YAML format file to import with.')
+  c.option('--headers STRING', 'A comma-delimited list of headers to use.')
+  c.option('--append', 'Appends to an existing database if one exists.')
+  c.option('--overwrite', 'Overwrites an existing database if one exists.')
+  c.when_called do|args, options|
+    abort("You cannot specify --append and --overwrite at the same time.") if options.append && options.overwrite
+
+    # Check if the server is running.
+    if !SkyDB.ping
+      puts "Sky is not currently running on #{SkyDB.client.host}:#{SkyDB.client.port}."
+      exit(1)
+    end
+
+    # Setup importer.
+    importer = SkyDB::Import::Importer.new()
+    importer.table_name = options.table || ask("Table: ")
+    importer.headers = options.headers.nil? ? nil : options.headers.split(/,/)
+    
+    # Load transform files by name.
+    formats = options.format || ask("Format: ")
+    formats.split(',').each do |format|
+      begin
+        importer.load_transform_file(format)
+      rescue SkyDB::Import::Importer::TransformNotFound => e
+        puts "ERROR: #{e.message}\n\n"
+        exit(1)
+      end
+    end
+    
+    # Check if table exists on server already.
+    table = SkyDB.get_table(importer.table_name)
+
+    # If table does not exists, then ask if it should be created.
+    if table.nil?
+      print "'#{importer.table_name}' does not exist. Create it? [Yn] "
+      c = STDIN.gets.chomp.upcase
+      exit(0) unless  c == '' || c[0] == 'Y'
+      SkyDB.create_table(SkyDB::Table.new(importer.table_name))
+    
+    # If it does exist, ask if the import should append.
+    else
+      if !options.append
+        c = nil
+        if options.overwrite
+          c = 'O'
+        else
+          print "'#{importer.table_name}' already exists. Append or overwrite? [Ao] "
+          c = STDIN.gets.chomp.upcase[0]
+        end
+        
+        if c == 'O'
+          SkyDB.delete_table(SkyDB::Table.new(importer.table_name))
+          SkyDB.create_table(SkyDB::Table.new(importer.table_name))
+        end
+      end
+    end
+    
+    # Import!
+    importer.import(args)
+  end
+end

data/lib/ext/hash.rb

@@ -0,0 +1,11 @@
+class Hash
+  # Performs a deep conversion of string keys to symbol keys.
+  def _symbolize_keys!
+    keys.select {|key| key.is_a?(String)}.each do |key|
+      self[key]._symbolize_keys! if self[key].is_a?(Hash)
+      self[(key.to_sym rescue key) || key] = self.delete(key)
+    end
+
+    return self
+  end
+end

data/lib/ext/treetop.rb

@@ -0,0 +1,19 @@
+module Treetop
+  # Searches the syntax node hierarchy for elements that match a given class.
+  def self.search(node, type)
+    # If this is a matching node then return it.
+    if node.is_a?(type)
+      return [node]
+
+    # Otherwise search children.
+    elsif !node.elements.nil?
+      ret = []
+      node.elements.each do |element|
+        ret = ret.concat(Treetop.search(element, type))
+      end
+      return ret
+    end
+
+    return []
+  end
+end

data/lib/skydb.rb

@@ -1,16 +1,21 @@
 require 'date'
 require 'msgpack'
 require 'socket'
+require 'treetop'
 
 require 'skydb/action'
 require 'skydb/client'
 require 'skydb/event'
 require 'skydb/message'
 require 'skydb/property'
+require 'skydb/query'
+require 'skydb/table'
 require 'skydb/timestamp'
 require 'skydb/version'
 
+require 'ext/hash'
 require 'ext/string'
+require 'ext/treetop'
 
 class SkyDB
   ############################################################################
@@ -32,13 +37,15 @@ class SkyDB
 
   CLIENT_PASSTHROUGH = [
     :host, :host=, :port, :port=,
-    :table, :table=,
-    :multi, :ping,
+    :table_name, :table_name=,
+    :multi, :ping, :lookup,
     :add_event,
+    :create_table, :delete_table, :get_table,
     :add_action, :get_action, :get_actions,
     :add_property, :get_property, :get_properties,
     :next_actions,
-    :aggregate
+    :aggregate,
+    :select
   ]
   
   

data/lib/skydb/client.rb

@@ -21,6 +21,8 @@ class SkyDB
 
     # Initializes the client.
     def initialize(options={})
+      @multi_message_max_count = 0
+      
       self.host = options[:host] || DEFAULT_HOST
       self.port = options[:port] || DEFAULT_PORT
     end
@@ -38,8 +40,8 @@ class SkyDB
     # The port on the host to connect to.
     attr_accessor :port
 
-    # The table to connect to.
-    attr_accessor :table
+    # The name of the table to connect to.
+    attr_accessor :table_name
 
 
     ##########################################################################
@@ -48,6 +50,33 @@ class SkyDB
     #
     ##########################################################################
     
+    ####################################
+    # Table Messages
+    ####################################
+
+    # Creates a table on the server.
+    #
+    # @param [Table] table  the table to add.
+    def create_table(table, options={})
+      return send_message(SkyDB::Message::CreateTable.new(table, options))
+    end
+
+    # Deletes a table on the server.
+    #
+    # @param [Table] table  the table to delete.
+    def delete_table(table, options={})
+      return send_message(SkyDB::Message::DeleteTable.new(table, options))
+    end
+
+    # Retrieves an individual table from the server, if it exists. Otherwise
+    # returns nil.
+    #
+    # @param [Fixnum] action_id  the identifier of the action to retrieve.
+    def get_table(action_id, options={})
+      return send_message(SkyDB::Message::GetTable.new(action_id, options))
+    end
+
+
     ####################################
     # Action Messages
     ####################################
@@ -122,7 +151,7 @@ class SkyDB
 
 
     ####################################
-    # Ping
+    # Utility message
     ####################################
 
     # Checks if the server is up and running.
@@ -137,6 +166,12 @@ class SkyDB
       end
     end
 
+    # Looks up lists of actions and properties by name.
+    def lookup(options={})
+      send_message(SkyDB::Message::Lookup.new(options))
+      return nil
+    end
+
 
     ####################################
     # Lua Messages
@@ -150,6 +185,18 @@ class SkyDB
     end
 
 
+    ####################################
+    # Query Interface
+    ####################################
+
+    # Starts a query against the database.
+    #
+    # @param [String] selection  a list of properties to select from the database.
+    def select(fields)
+      return SkyDB::Query.new(:client => self).select(fields)
+    end
+
+
     ####################################
     # Multi message
     ####################################
@@ -160,17 +207,16 @@ class SkyDB
       
       # Create multi-message.
       @multi_message = SkyDB::Message::Multi.new(options)
+      @multi_message_max_count = options[:max_count].to_i
       
       # Execute the block normally and send the message.
       begin
         yield
         
-        # Clear multi message so it doesn't add to itself.
-        tmp = @multi_message
-        @multi_message = nil
-        
         # Send all messages at once.
-        send_message(tmp)
+        if @multi_message.messages.length > 0
+          send_message(@multi_message)
+        end
 
       ensure
         @multi_message = nil
@@ -190,43 +236,61 @@ class SkyDB
     # @return [Object]  the object returned by the server.
     def send_message(message)
       # Set the table if they're not set.
-      message.table = table if message.table.nil? || message.table.empty?
+      message.table_name = table_name if message.table_name.nil? || message.table_name.empty?
 
       # Validate message before sending.
       message.validate!
       
       # If this is part of a multi message then simply append the message for
       # later sending.
-      if !@multi_message.nil?
+      if !@multi_message.nil? && @multi_message != message
         @multi_message.messages << message
+        
+        # Send off the MULTI if the message count is above our limit.
+        if @multi_message_max_count > 0 && @multi_message.messages.length >= @multi_message_max_count
+          send_message(@multi_message)
+          @multi_message = SkyDB::Message::Multi.new()
+        end
+        
         return nil
       
       # Otherwise send the message immediately.
       else
-        # Connect to the server.
-        socket = TCPSocket.new(host, port.to_i)
+        begin
+          # Connect to the server.
+          socket = TCPSocket.new(host, port.to_i)
       
-        # Encode and send message request.
-        message.encode(socket)
+          # Encode and send message request.
+          message.encode(socket)
       
-        # Decode msgpack response. There should only be one return object.
-        response = nil
-        unpacker = MessagePack::Unpacker.new(socket)
-        unpacker.each do |obj|
-          response = obj
-          break
-        end
+          # Retrieve the respose as a buffer so we can inspect it.
+          #msg, x = *socket.recvmsg
+          #buffer = StringIO.new(msg)
+          #puts "[#{message.message_name}]< #{buffer.string.to_hex}" if SkyDB.debug
+      
+          # Decode msgpack response. There should only be one return object.
+          response = nil
+          unpacker = MessagePack::Unpacker.new(socket)
+          unpacker.each do |obj|
+            response = obj
+            break
+          end
       
-        # Close socket.
-        socket.close()
+          # Close socket.
+          socket.close()
       
-        # TODO: Exception processing.
+          # TODO: Exception processing.
       
-        # Process response back through the message.
-        response = message.process_response(response)
+          # Process response back through the message.
+          response = message.process_response(response)
       
-        # Return response.
-        return response
+          # Return response.
+          return response
+
+        ensure
+          # Make sure we remove the multi-message if that's what we're sending.
+          @multi_message = nil if @multi_message == message
+        end
       end
     end
   end

data/lib/skydb/import.rb

@@ -0,0 +1,7 @@
+require 'skydb/import/importer'
+require 'skydb/import/translator'
+
+class SkyDB
+  class Import
+  end
+end

data/lib/skydb/import/importer.rb

@@ -0,0 +1,258 @@
+require 'yaml'
+require 'csv'
+require 'ruby-progressbar'
+
+class SkyDB
+  class Import
+    class Importer
+      ##########################################################################
+      #
+      # Errors
+      #
+      ##########################################################################
+
+      class TransformNotFound < StandardError; end
+      
+
+      ##########################################################################
+      #
+      # Constructor
+      #
+      ##########################################################################
+
+      # Initializes the importer.
+      def initialize(options={})
+        @translators = []
+
+        self.client = options[:client] || SkyDB.client
+        self.table_name  = options[:table_name]
+        self.format = options[:format]
+        self.files  = options[:files] || []
+      end
+    
+
+      ##########################################################################
+      #
+      # Attributes
+      #
+      ##########################################################################
+
+      # The client to access the Sky server with.
+      attr_accessor :client
+
+      # The name of the table to import into.
+      attr_accessor :table_name
+
+      # The format file to use for translating the input data.
+      attr_accessor :format
+
+      # A list of translators to use to convert input rows into output rows.
+      attr_reader :translators
+
+      # A list of files to input from.
+      attr_accessor :files
+
+      # A list of header names to use for CSV files. Using this option will
+      # treat the CSV input as not having a header row.
+      attr_accessor :headers
+
+
+      ##########################################################################
+      #
+      # Methods
+      #
+      ##########################################################################
+    
+      ##################################
+      # Import
+      ##################################
+    
+      # Imports the rows from a list of files.
+      #
+      # @param [Array]  a list of files to import.
+      def import(files)
+        files = [files] unless files.is_a?(Array)
+        
+        # Set the table to import into.
+        SkyDB.table_name = table_name
+        
+        # Loop over each of the files.
+        files.each do |file|
+          # Initialize progress bar.
+          count = %x{wc -l #{file}}.split.first.to_i
+          progress_bar = ::ProgressBar.create(
+            :total => count,
+            :format => ('%-40s' % file) + ' |%B| %P%%'
+          )
+
+          # Determine column separator by extension.
+          col_sep = ','
+          if File.extname(file) == '.tsv' || File.extname(file) == '.txt'
+            col_sep = "\t"
+          end
+
+          file = File.open(file, 'r')
+          begin
+            SkyDB.multi(:max_count => 1000) do
+              # Process each line of the CSV file.
+              CSV.foreach(file, :headers => headers.nil?, :col_sep => col_sep) do |row|
+                input = {}
+                
+                # If headers were not specified then use the ones from the
+                # CSV file and just convert the row to a hash.
+                if headers.nil?
+                  input = row.to_hash
+                
+                # If headers were specified then manually convert the row
+                # using the headers provided.
+                else
+                  headers.each_with_index do |header, index|
+                    input[header] = row[index]
+                  end
+                end
+                
+                # Convert input line to a symbolized hash.
+                output = translate(input)
+                output._symbolize_keys!
+                
+                # p output
+                
+                # Convert hash to an event and send to Sky.
+                event = SkyDB::Event.new(output)
+
+                if !(event.object_id > 0)
+                  progress_bar.clear()
+                  puts "[ERROR] Invalid object id on line #{$.}."
+                elsif event.timestamp.nil?
+                  progress_bar.clear()
+                  puts "[ERROR] Invalid timestamp on line #{$.}."
+                else
+                  SkyDB.add_event(event)
+                end
+              
+                # Update progress bar.
+                progress_bar.increment()
+              end
+            end
+          ensure
+            file.close
+          end
+
+          # Finish progress bar.
+          progress_bar.finish()        
+        end
+        
+        return nil
+      end
+
+
+      ##################################
+      # Translation
+      ##################################
+
+      # Translates an input hash into an output hash using the translators.
+      #
+      # @param [Hash]  the input hash.
+      #
+      # @return [Hash]  the output hash.
+      def translate(input)
+        output = {}
+
+        translators.each do |translator|
+          translator.translate(input, output)
+        end
+
+        return output
+      end
+      
+      
+      ##################################
+      # Transform Management
+      ##################################
+    
+      # Parses and appends the contents of a transform file to the importer.
+      #
+      # @param [String]  the YAML formatted transform file.
+      def load_transform(content)
+        # Parse the transform file.
+        transform = {'fields' => {}}.merge(YAML.load(content))
+
+        # Load any libraries requested by the format file.
+        if transform['require'].is_a?(Array)
+          transform['require'].each do |library_name|
+            require library_name
+          end
+        end
+
+        # Load individual field translations.
+        load_transform_fields(transform['fields'])
+        
+        # Load a free-form translate function if specified.
+        if !transform['translate'].nil?
+          @translators << Translator.new(
+            :translate_function => transform['translate']
+          )
+        end
+        
+        return nil
+      end
+
+      # Loads a hash of transforms.
+      #
+      # @param [Hash]  the hash of transform info.
+      # @param [Array]  the path of fields.
+      def load_transform_fields(fields, path=nil)
+        
+        # Convert each field to a translator.
+        fields.each_pair do |key, value|
+          translator = Translator.new(:output_field => (path.nil? ? key : path.clone.concat([key])))
+
+          # Load a regular transform.
+          if value.is_a?(String)
+            # If the line is wrapped in curly braces then generate a translate function.
+            m, code = *value.match(/^\s*\{(.*)\}\s*$/)
+            if !m.nil?
+              translator.translate_function = code
+          
+            # Otherwise it's a colon-separated field describing the input field and data type.
+            else
+              input_field, format = *value.strip.split(":")
+              translator.input_field = input_field
+              translator.format = format
+            end
+
+          # If this field is a hash then load it as a nested transform.
+          elsif value.is_a?(Hash)
+            load_transform_fields(value, path.to_a.clone.flatten.concat([key]))
+          
+          else
+            raise "Invalid data type for '#{key}' in transform file: #{value.class}"
+          end
+          
+          # Append to the list of translators.
+          @translators << translator
+        end
+      end
+
+
+      # Parses and appends the contents of a transform file to the importer.
+      #
+      # @param [String]  the filename to load from.
+      def load_transform_file(filename)
+        transforms_path = File.expand_path(File.join(File.dirname(__FILE__), 'transforms'))
+        named_transform_path = File.join(transforms_path, "#{filename}.yml")
+        
+        # If it's just a word then find it in the gem.
+        if filename.index(/^\w+$/)
+          raise TransformNotFound.new("Named transform not available: #{filename} (#{named_transform_path})") unless File.exists?(named_transform_path)
+          return load_transform(IO.read(named_transform_path))
+
+        # Otherwise load it from the present working directory.
+        else
+          raise TransformNotFound.new("Transform file not found: #{filename}") unless File.exists?(filename)
+          return load_transform(IO.read(filename))
+        end
+      end
+    end
+  end
+end