skydb 0.2.2 → 0.2.3

data/test/{event_test.rb → unit/event_test.rb}

@@ -15,11 +15,6 @@ class TestEvent < MiniTest::Unit::TestCase
     assert_equal 12, @event.object_id
   end
   
-  def test_invalid_object_id
-    @event.object_id = "foo"
-    assert_equal 0, @event.object_id
-  end
-  
 
   ######################################
   # Timestamp

data/test/unit/import/importer_test.rb

@@ -0,0 +1,208 @@
+require 'test_helper'
+
+class TestImporter < MiniTest::Unit::TestCase
+  def setup
+    @importer = SkyDB::Import::Importer.new(:table_name => 'test')
+    @input  = {
+      'myString' => 'Hello',
+      'foo' => 100
+    }
+    @output = {}
+  end
+  
+
+  ######################################
+  # Transform File
+  ######################################
+
+  def test_load_simple_transform
+    @importer.load_transform(
+      <<-BLOCK.unindent
+      fields:
+        name: myString
+      BLOCK
+    )
+    
+    assert_equal "myString", @importer.translators.first.input_field
+    assert_equal "name", @importer.translators.first.output_field
+    assert_equal "string", @importer.translators.first.format
+  end
+
+  def test_load_simple_proc
+    @importer.load_transform(
+      <<-BLOCK.unindent
+      fields:
+        name: "{ output['foo'] * 10 }"
+      BLOCK
+    )
+    
+    assert_equal "name", @importer.translators.first.output_field
+    assert !@importer.translators.first.translate_function.nil?
+  end
+
+
+  ######################################
+  # Import
+  ######################################
+
+  def test_import_csv
+    out, err = capture_io do
+      events = [mock(), mock(), mock(), mock()]
+      SkyDB::Event.expects(:new).with(:object_id => '100', :timestamp => Chronic.parse("2000-01-01T00:00:00Z"), :action => {:name => '/index.html'}).returns(events[0])
+      SkyDB::Event.expects(:new).with(:object_id => '100', :timestamp => Chronic.parse("2000-01-01T00:01:00Z"), :action => {:name => '/signup.html'}).returns(events[1])
+      SkyDB::Event.expects(:new).with(:object_id => '100', :timestamp => Chronic.parse("2000-01-01T00:02:00Z"), :action => {:name => '/buy.html'}).returns(events[2])
+      SkyDB::Event.expects(:new).with(:object_id => '101', :timestamp => Chronic.parse("2000-01-02T12:00:00Z"), :action => {:name => '/index.html'}).returns(events[3])
+      SkyDB.expects(:add_event).with(events[0])
+      SkyDB.expects(:add_event).with(events[1])
+      SkyDB.expects(:add_event).with(events[2])
+      SkyDB.expects(:add_event).with(events[3])
+      @importer.load_transform_file('sky')
+      @importer.import(['fixtures/unit/importer/simple.csv'], :progress_bar => false)
+    end
+    assert_equal '', out
+    assert_equal '[import] Determining file type: csv', err.chomp
+  end
+
+  def test_import_tsv
+    out, err = capture_io do
+      events = [mock(), mock(), mock(), mock()]
+      SkyDB::Event.expects(:new).with(:object_id => '100', :timestamp => Chronic.parse("2000-01-01T00:00:00Z"), :action => {:name => '/index.html'}).returns(events[0])
+      SkyDB::Event.expects(:new).with(:object_id => '100', :timestamp => Chronic.parse("2000-01-01T00:01:00Z"), :action => {:name => '/signup.html'}).returns(events[1])
+      SkyDB::Event.expects(:new).with(:object_id => '100', :timestamp => Chronic.parse("2000-01-01T00:02:00Z"), :action => {:name => '/buy.html'}).returns(events[2])
+      SkyDB::Event.expects(:new).with(:object_id => '101', :timestamp => Chronic.parse("2000-01-02T12:00:00Z"), :action => {:name => '/index.html'}).returns(events[3])
+      SkyDB.expects(:add_event).with(events[0])
+      SkyDB.expects(:add_event).with(events[1])
+      SkyDB.expects(:add_event).with(events[2])
+      SkyDB.expects(:add_event).with(events[3])
+      @importer.load_transform_file('sky')
+      @importer.import(['fixtures/unit/importer/simple.tsv'], :progress_bar => false)
+    end
+    assert_equal '', out
+    assert_equal '[import] Determining file type: tsv', err.chomp
+  end
+
+  def test_import_json
+    out, err = capture_io do
+      events = [mock(), mock(), mock(), mock()]
+      SkyDB::Event.expects(:new).with(:object_id => '100', :timestamp => Chronic.parse("2000-01-01T00:00:00Z"), :action => {:name => '/index.html'}).returns(events[0])
+      SkyDB::Event.expects(:new).with(:object_id => '100', :timestamp => Chronic.parse("2000-01-01T00:01:00Z"), :action => {:name => '/signup.html'}).returns(events[1])
+      SkyDB::Event.expects(:new).with(:object_id => '100', :timestamp => Chronic.parse("2000-01-01T00:02:00Z"), :action => {:name => '/buy.html'}).returns(events[2])
+      SkyDB::Event.expects(:new).with(:object_id => '101', :timestamp => Chronic.parse("2000-01-02T12:00:00Z"), :action => {:name => '/index.html'}).returns(events[3])
+      SkyDB.expects(:add_event).with(events[0])
+      SkyDB.expects(:add_event).with(events[1])
+      SkyDB.expects(:add_event).with(events[2])
+      SkyDB.expects(:add_event).with(events[3])
+      @importer.load_transform_file('sky')
+      @importer.import(['fixtures/unit/importer/simple.json'], :progress_bar => false)
+    end
+    assert_equal '', out
+    assert_equal '[import] Determining file type: json', err.chomp
+  end
+
+  def test_import_apache_log
+    out, err = capture_io do
+      events = [mock(), mock(), mock(), mock(), mock()]
+      SkyDB::Event.expects(:new).with(:object_id => '66.29.187.16', :timestamp => DateTime.parse("2013-01-13T04:05:07+00:00"), :action => {:name => "/users/1"}).returns(events[0])
+      SkyDB::Event.expects(:new).with(:object_id => '70.193.12.4', :timestamp => DateTime.parse("2013-01-13T04:05:09+00:00"), :action => {:name => "/tweets/new"}).returns(events[1])
+      SkyDB::Event.expects(:new).with(:object_id => '67.228.32.10', :timestamp => DateTime.parse("2013-01-13T04:05:07+00:00"), :action => {:name => "/tweets"}).returns(events[2])
+      SkyDB::Event.expects(:new).with(:object_id => '157.55.35.85', :timestamp => DateTime.parse("2013-01-13T04:05:07+00:00"), :action => {:name => "/index.html"}).returns(events[3])
+      SkyDB::Event.expects(:new).with(:object_id => '66.29.187.16', :timestamp => DateTime.parse("2013-01-13T04:05:07+00:00"), :action => {:name => "/messages/1840832/open"}).returns(events[4])
+      SkyDB.expects(:add_event).with(events[0])
+      SkyDB.expects(:add_event).with(events[1])
+      SkyDB.expects(:add_event).with(events[2])
+      SkyDB.expects(:add_event).with(events[3])
+      SkyDB.expects(:add_event).with(events[4])
+      @importer.load_transform_file('apache')
+      @importer.import(['fixtures/unit/importer/simple.log'], :progress_bar => false)
+    end
+    assert_equal '', out
+    assert_equal '[import] Determining file type: apache_log', err.chomp
+  end
+
+  def test_import_bzip2_apache_log
+    out, err = capture_io do
+      event = mock()
+      SkyDB::Event.expects(:new).with(:object_id => '66.29.187.16', :timestamp => DateTime.parse("2013-01-13T04:05:07+00:00"), :action => {:name => "/users/1"}).returns(event)
+      SkyDB.expects(:add_event).with(event)
+      @importer.load_transform_file('apache')
+      @importer.file_type = :apache_log
+      @importer.import(['fixtures/unit/importer/simple.log.bz2'])
+    end
+    assert_equal '', out
+    assert_equal '', err
+  end
+
+  def test_import_gzip_apache_log
+    out, err = capture_io do
+      event = mock()
+      SkyDB::Event.expects(:new).with(:object_id => '66.29.187.16', :timestamp => DateTime.parse("2013-01-13T04:05:07+00:00"), :action => {:name => "/users/1"}).returns(event)
+      SkyDB.expects(:add_event).with(event)
+      @importer.load_transform_file('apache')
+      @importer.file_type = :apache_log
+      @importer.import(['fixtures/unit/importer/simple.log.gz'])
+    end
+    assert_equal '', out
+    assert_equal '', err
+  end
+
+  def test_import_override_file_type
+    out, err = capture_io do
+      event = mock()
+      SkyDB::Event.expects(:new).with(:object_id => '100', :timestamp => Chronic.parse("2000-01-01T00:00:00Z"), :action => {:name => '/index.html'}).returns(event)
+      SkyDB.expects(:add_event).with(event)
+      @importer.load_transform_file('sky')
+      @importer.file_type = :csv
+      @importer.import(['fixtures/unit/importer/simple'], :progress_bar => false)
+    end
+    assert_equal '', out
+    assert_equal '', err
+  end
+
+  def test_import_csv_no_headers
+    out, err = capture_io do
+      events = [mock(), mock(), mock(), mock()]
+      SkyDB::Event.expects(:new).with(:object_id => '100', :timestamp => Chronic.parse("2000-01-01T00:00:00Z"), :action => {:name => '/index.html'}).returns(events[0])
+      SkyDB.expects(:add_event).with(events[0])
+      @importer.headers = ['object_id', 'timestamp', 'action.name']
+      @importer.load_transform_file('sky')
+      @importer.import(['fixtures/unit/importer/no_headers.csv'], :progress_bar => false)
+    end
+    assert_equal '', out
+    assert_equal '[import] Determining file type: csv', err.chomp
+  end
+
+  def test_import_bad_timestamp
+    out, err = capture_io do
+      events = mock()
+      SkyDB::Event.expects(:new).never
+      SkyDB.expects(:add_event).never
+      @importer.load_transform_file('sky')
+      @importer.import(['fixtures/unit/importer/bad_timestamp.csv'], :progress_bar => false)
+    end
+    assert_equal '', out
+    assert_equal(
+      "[import] Determining file type: csv\n" +
+      "[ERROR] Invalid timestamp on line 2",
+      err.chomp
+    )
+  end
+  
+  def test_import_unsupported_file_type
+    e = nil
+    out, err = capture_io do
+      e = assert_raises(SkyDB::Import::Importer::UnsupportedFileType) do
+        @importer.import(['fixtures/unit/importer/unsupported.xxx'], :progress_bar => false)
+      end
+    end
+    assert_equal 'File type not supported by importer: .xxx', e.message
+    assert_equal '', out
+    assert_equal "[import] Determining file type: ???", err.chomp
+  end
+  
+  def test_import_transform_not_found
+    e = assert_raises(SkyDB::Import::Importer::TransformNotFound) do
+      @importer.load_transform_file('no_such_transform.yml')
+    end
+    assert_equal 'Transform file not found: no_such_transform.yml', e.message
+  end
+end

data/test/{message → unit/message}/add_event_message_test.rb

@@ -14,7 +14,7 @@ class TestMessageAddEvent < MiniTest::Unit::TestCase
     buffer = StringIO.new
     @message.table_name = "users"
     @message.event = SkyDB::Event.new(
-      object_id: 12,
+      object_id: "12",
       timestamp:DateTime.parse('2010-01-02T10:30:20Z'),
       action: {
         name:"/index.html",
@@ -30,6 +30,6 @@ class TestMessageAddEvent < MiniTest::Unit::TestCase
       }
     )
     @message.encode(buffer)
-    assert_bytes "\x93\x01\xa9add_event\xa5users\x84\xa8objectId\x0c\xa9timestamp\xcf\x00\x04\x7c\x2b\xf9\x9b\x87\x00\xa6action\x83\xa4name\xab\x2findex\x2ehtml\xa7astring\xa3foo\xa4aint\x14\xa4data\x85\xa7ostring\xa3bar\xa4oint\x0a\xa7odouble\xcb\x40Y\x06fffff\xa5otrue\xc3\xa6ofalse\xc2", buffer
+    assert_bytes "\x93\x01\xa9add_event\xa5users\x84\xa8objectId\xa3\xa212\xa9timestamp\xcf\x00\x04\x7c\x2b\xf9\x9b\x87\x00\xa6action\x83\xa4name\xab\x2findex\x2ehtml\xa7astring\xa3foo\xa4aint\x14\xa4data\x85\xa7ostring\xa3bar\xa4oint\x0a\xa7odouble\xcb\x40Y\x06fffff\xa5otrue\xc3\xa6ofalse\xc2", buffer
   end
 end

data/test/unit/message/get_tables_message_test.rb

@@ -0,0 +1,18 @@
+# encoding: binary
+require 'test_helper'
+
+class TestMessageGetTables < MiniTest::Unit::TestCase
+  def setup
+    @message = SkyDB::Message::GetTables.new()
+  end
+  
+  ######################################
+  # Encoding
+  ######################################
+
+  def test_encode
+    buffer = StringIO.new
+    @message.encode(buffer)
+    assert_bytes "\x93\x01\xaaget_tables\xa0", buffer
+  end
+end

data/test/unit/query/after_test.rb

@@ -0,0 +1,89 @@
+require 'test_helper'
+
+class TestQueryAfterCondition < MiniTest::Unit::TestCase
+  ##############################################################################
+  #
+  # Setup / Teardown
+  #
+  ##############################################################################
+
+  def setup
+    @after = SkyDB::Query::AfterCondition.new()
+  end
+
+
+  ##############################################################################
+  #
+  # Tests
+  #
+  ##############################################################################
+
+  ######################################
+  # Validation
+  ######################################
+
+  def test_validate_action
+    e = assert_raises(SkyDB::Query::ValidationError) do
+      SkyDB::Query::AfterCondition.new(:function_name => "foo").validate!
+    end
+    assert_match /^Action with non-zero identifier required/, e.message
+  end
+
+  def test_validate_enter_action
+    e = assert_raises(SkyDB::Query::ValidationError) do
+      SkyDB::Query::AfterCondition.new(:action => :enter, :function_name => "foo").validate!
+    end
+    assert_equal "Enter actions cannot be used with an 'after' condition. Please use an 'on' condition instead.", e.message
+  end
+  
+  def test_validate_function_name
+    e = assert_raises(SkyDB::Query::ValidationError) do
+      SkyDB::Query::AfterCondition.new(:action => 10).validate!
+    end
+    assert_match /^Invalid function name ''/, e.message
+  end
+  
+  
+  ######################################
+  # Code Generation
+  ######################################
+
+  def test_codegen
+    @after = SkyDB::Query::AfterCondition.new(:action => 10, :function_name => "foo")
+    expected =
+      <<-BLOCK.unindent
+        function foo(cursor, data)
+          if cursor:eos() or cursor:eof() then return false end
+          repeat
+            if cursor.event.action_id == 10 then
+              cursor:next()
+              return true
+            end
+          until not cursor:next()
+          return false
+        end
+      BLOCK
+    assert_equal expected, @after.codegen()
+  end
+
+  def test_codegen_within_1_step
+    @after = SkyDB::Query::AfterCondition.new(:action => 10, :within => {:quantity => 1, :unit => 'step'}, :function_name => "foo")
+    expected =
+      <<-BLOCK.unindent
+        function foo(cursor, data)
+          if cursor:eos() or cursor:eof() then return false end
+          remaining = 1
+          repeat
+            if remaining <= 0 then return false end
+            if cursor.event.action_id == 10 then
+              cursor:next()
+              return true
+            end
+            remaining = remaining - 1
+          until not cursor:next()
+          return false
+        end
+      BLOCK
+    assert_equal expected, @after.codegen()
+  end
+end

data/test/{query/after_test.rb → unit/query/on_test.rb}

@@ -1,6 +1,6 @@
 require 'test_helper'
 
-class TestQueryAfter < MiniTest::Unit::TestCase
+class TestQueryOnCondition < MiniTest::Unit::TestCase
   ##############################################################################
   #
   # Setup / Teardown
@@ -8,7 +8,7 @@ class TestQueryAfter < MiniTest::Unit::TestCase
   ##############################################################################
 
   def setup
-    @after = SkyDB::Query::After.new()
+    @condition = SkyDB::Query::OnCondition.new()
   end
 
 
@@ -24,14 +24,14 @@ class TestQueryAfter < MiniTest::Unit::TestCase
 
   def test_validate_action
     e = assert_raises(SkyDB::Query::ValidationError) do
-      SkyDB::Query::After.new(:function_name => "foo").validate!
+      SkyDB::Query::OnCondition.new(:function_name => "foo").validate!
     end
     assert_match /^Action with non-zero identifier required/, e.message
   end
-  
+
   def test_validate_function_name
     e = assert_raises(SkyDB::Query::ValidationError) do
-      SkyDB::Query::After.new(:action => 10).validate!
+      SkyDB::Query::OnCondition.new(:action => 10).validate!
     end
     assert_match /^Invalid function name ''/, e.message
   end
@@ -42,30 +42,30 @@ class TestQueryAfter < MiniTest::Unit::TestCase
   ######################################
 
   def test_codegen
-    @after = SkyDB::Query::After.new(:action => 10, :function_name => "foo")
+    @condition = SkyDB::Query::OnCondition.new(:action => 10, :function_name => "foo")
     expected =
       <<-BLOCK.unindent
         function foo(cursor, data)
+          if cursor:eos() or cursor:eof() then return false end
           repeat
             if cursor.event.action_id == 10 then
-              cursor:next()
               return true
             end
           until not cursor:next()
           return false
         end
       BLOCK
-    assert_equal expected, @after.codegen()
+    assert_equal expected, @condition.codegen()
   end
 
   def test_codegen_enter
-    @after = SkyDB::Query::After.new(:action => :enter, :function_name => "foo")
+    @condition = SkyDB::Query::OnCondition.new(:action => :enter, :function_name => "foo")
     expected =
       <<-BLOCK.unindent
         function foo(cursor, data)
           return (cursor.session_event_index == 0)
         end
       BLOCK
-    assert_equal expected, @after.codegen()
+    assert_equal expected, @condition.codegen()
   end
 end