skydb 0.2.2 → 0.2.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. data/bin/sky +4 -0
  2. data/lib/skydb.rb +3 -2
  3. data/lib/skydb/action.rb +19 -0
  4. data/lib/skydb/client.rb +15 -5
  5. data/lib/skydb/event.rb +3 -7
  6. data/lib/skydb/import/importer.rb +236 -59
  7. data/lib/skydb/import/transforms/apache.yml +4 -0
  8. data/lib/skydb/import/transforms/sky.yml +20 -12
  9. data/lib/skydb/message.rb +1 -0
  10. data/lib/skydb/message/add_event.rb +1 -1
  11. data/lib/skydb/message/get_actions.rb +4 -0
  12. data/lib/skydb/message/get_properties.rb +4 -0
  13. data/lib/skydb/message/get_tables.rb +43 -0
  14. data/lib/skydb/message/lua/aggregate.rb +4 -0
  15. data/lib/skydb/property.rb +10 -0
  16. data/lib/skydb/query.rb +44 -59
  17. data/lib/skydb/query/after_condition.rb +104 -0
  18. data/lib/skydb/query/{after.rb → condition.rb} +37 -27
  19. data/lib/skydb/query/on_condition.rb +53 -0
  20. data/lib/skydb/query/selection.rb +131 -1
  21. data/lib/skydb/query/selection_field.rb +25 -0
  22. data/lib/skydb/query/selection_group.rb +21 -0
  23. data/lib/skydb/table.rb +7 -0
  24. data/lib/skydb/version.rb +1 -1
  25. data/test/integration/query_test.rb +102 -0
  26. data/test/test_helper.rb +42 -1
  27. data/test/{client_test.rb → unit/client_test.rb} +0 -0
  28. data/test/{event_test.rb → unit/event_test.rb} +0 -5
  29. data/test/unit/import/importer_test.rb +208 -0
  30. data/test/{import → unit/import}/translator_test.rb +0 -0
  31. data/test/{message → unit/message}/add_action_message_test.rb +0 -0
  32. data/test/{message → unit/message}/add_event_message_test.rb +2 -2
  33. data/test/{message → unit/message}/add_property_message_test.rb +0 -0
  34. data/test/{message → unit/message}/create_table_message_test.rb +0 -0
  35. data/test/{message → unit/message}/delete_table_message_test.rb +0 -0
  36. data/test/{message → unit/message}/get_action_message_test.rb +0 -0
  37. data/test/{message → unit/message}/get_actions_message_test.rb +0 -0
  38. data/test/{message → unit/message}/get_properties_message_test.rb +0 -0
  39. data/test/{message → unit/message}/get_property_message_test.rb +0 -0
  40. data/test/{message → unit/message}/get_table_message_test.rb +0 -0
  41. data/test/unit/message/get_tables_message_test.rb +18 -0
  42. data/test/{message → unit/message}/lookup_message_test.rb +0 -0
  43. data/test/{message → unit/message}/lua_aggregate_message_test.rb +0 -0
  44. data/test/{message → unit/message}/multi_message_test.rb +0 -0
  45. data/test/{message → unit/message}/next_action_message_test.rb +0 -0
  46. data/test/{message → unit/message}/ping_message_test.rb +0 -0
  47. data/test/{message_test.rb → unit/message_test.rb} +0 -0
  48. data/test/unit/query/after_test.rb +89 -0
  49. data/test/{query/after_test.rb → unit/query/on_test.rb} +10 -10
  50. data/test/{query → unit/query}/selection_test.rb +2 -2
  51. data/test/{query_test.rb → unit/query_test.rb} +32 -6
  52. data/test/{skydb_test.rb → unit/skydb_test.rb} +0 -0
  53. metadata +165 -53
  54. data/test/import/importer_test.rb +0 -42
File without changes
@@ -15,11 +15,6 @@ class TestEvent < MiniTest::Unit::TestCase
15
15
  assert_equal 12, @event.object_id
16
16
  end
17
17
 
18
- def test_invalid_object_id
19
- @event.object_id = "foo"
20
- assert_equal 0, @event.object_id
21
- end
22
-
23
18
 
24
19
  ######################################
25
20
  # Timestamp
@@ -0,0 +1,208 @@
1
+ require 'test_helper'
2
+
3
+ class TestImporter < MiniTest::Unit::TestCase
4
+ def setup
5
+ @importer = SkyDB::Import::Importer.new(:table_name => 'test')
6
+ @input = {
7
+ 'myString' => 'Hello',
8
+ 'foo' => 100
9
+ }
10
+ @output = {}
11
+ end
12
+
13
+
14
+ ######################################
15
+ # Transform File
16
+ ######################################
17
+
18
+ def test_load_simple_transform
19
+ @importer.load_transform(
20
+ <<-BLOCK.unindent
21
+ fields:
22
+ name: myString
23
+ BLOCK
24
+ )
25
+
26
+ assert_equal "myString", @importer.translators.first.input_field
27
+ assert_equal "name", @importer.translators.first.output_field
28
+ assert_equal "string", @importer.translators.first.format
29
+ end
30
+
31
+ def test_load_simple_proc
32
+ @importer.load_transform(
33
+ <<-BLOCK.unindent
34
+ fields:
35
+ name: "{ output['foo'] * 10 }"
36
+ BLOCK
37
+ )
38
+
39
+ assert_equal "name", @importer.translators.first.output_field
40
+ assert !@importer.translators.first.translate_function.nil?
41
+ end
42
+
43
+
44
+ ######################################
45
+ # Import
46
+ ######################################
47
+
48
+ def test_import_csv
49
+ out, err = capture_io do
50
+ events = [mock(), mock(), mock(), mock()]
51
+ SkyDB::Event.expects(:new).with(:object_id => '100', :timestamp => Chronic.parse("2000-01-01T00:00:00Z"), :action => {:name => '/index.html'}).returns(events[0])
52
+ SkyDB::Event.expects(:new).with(:object_id => '100', :timestamp => Chronic.parse("2000-01-01T00:01:00Z"), :action => {:name => '/signup.html'}).returns(events[1])
53
+ SkyDB::Event.expects(:new).with(:object_id => '100', :timestamp => Chronic.parse("2000-01-01T00:02:00Z"), :action => {:name => '/buy.html'}).returns(events[2])
54
+ SkyDB::Event.expects(:new).with(:object_id => '101', :timestamp => Chronic.parse("2000-01-02T12:00:00Z"), :action => {:name => '/index.html'}).returns(events[3])
55
+ SkyDB.expects(:add_event).with(events[0])
56
+ SkyDB.expects(:add_event).with(events[1])
57
+ SkyDB.expects(:add_event).with(events[2])
58
+ SkyDB.expects(:add_event).with(events[3])
59
+ @importer.load_transform_file('sky')
60
+ @importer.import(['fixtures/unit/importer/simple.csv'], :progress_bar => false)
61
+ end
62
+ assert_equal '', out
63
+ assert_equal '[import] Determining file type: csv', err.chomp
64
+ end
65
+
66
+ def test_import_tsv
67
+ out, err = capture_io do
68
+ events = [mock(), mock(), mock(), mock()]
69
+ SkyDB::Event.expects(:new).with(:object_id => '100', :timestamp => Chronic.parse("2000-01-01T00:00:00Z"), :action => {:name => '/index.html'}).returns(events[0])
70
+ SkyDB::Event.expects(:new).with(:object_id => '100', :timestamp => Chronic.parse("2000-01-01T00:01:00Z"), :action => {:name => '/signup.html'}).returns(events[1])
71
+ SkyDB::Event.expects(:new).with(:object_id => '100', :timestamp => Chronic.parse("2000-01-01T00:02:00Z"), :action => {:name => '/buy.html'}).returns(events[2])
72
+ SkyDB::Event.expects(:new).with(:object_id => '101', :timestamp => Chronic.parse("2000-01-02T12:00:00Z"), :action => {:name => '/index.html'}).returns(events[3])
73
+ SkyDB.expects(:add_event).with(events[0])
74
+ SkyDB.expects(:add_event).with(events[1])
75
+ SkyDB.expects(:add_event).with(events[2])
76
+ SkyDB.expects(:add_event).with(events[3])
77
+ @importer.load_transform_file('sky')
78
+ @importer.import(['fixtures/unit/importer/simple.tsv'], :progress_bar => false)
79
+ end
80
+ assert_equal '', out
81
+ assert_equal '[import] Determining file type: tsv', err.chomp
82
+ end
83
+
84
+ def test_import_json
85
+ out, err = capture_io do
86
+ events = [mock(), mock(), mock(), mock()]
87
+ SkyDB::Event.expects(:new).with(:object_id => '100', :timestamp => Chronic.parse("2000-01-01T00:00:00Z"), :action => {:name => '/index.html'}).returns(events[0])
88
+ SkyDB::Event.expects(:new).with(:object_id => '100', :timestamp => Chronic.parse("2000-01-01T00:01:00Z"), :action => {:name => '/signup.html'}).returns(events[1])
89
+ SkyDB::Event.expects(:new).with(:object_id => '100', :timestamp => Chronic.parse("2000-01-01T00:02:00Z"), :action => {:name => '/buy.html'}).returns(events[2])
90
+ SkyDB::Event.expects(:new).with(:object_id => '101', :timestamp => Chronic.parse("2000-01-02T12:00:00Z"), :action => {:name => '/index.html'}).returns(events[3])
91
+ SkyDB.expects(:add_event).with(events[0])
92
+ SkyDB.expects(:add_event).with(events[1])
93
+ SkyDB.expects(:add_event).with(events[2])
94
+ SkyDB.expects(:add_event).with(events[3])
95
+ @importer.load_transform_file('sky')
96
+ @importer.import(['fixtures/unit/importer/simple.json'], :progress_bar => false)
97
+ end
98
+ assert_equal '', out
99
+ assert_equal '[import] Determining file type: json', err.chomp
100
+ end
101
+
102
+ def test_import_apache_log
103
+ out, err = capture_io do
104
+ events = [mock(), mock(), mock(), mock(), mock()]
105
+ SkyDB::Event.expects(:new).with(:object_id => '66.29.187.16', :timestamp => DateTime.parse("2013-01-13T04:05:07+00:00"), :action => {:name => "/users/1"}).returns(events[0])
106
+ SkyDB::Event.expects(:new).with(:object_id => '70.193.12.4', :timestamp => DateTime.parse("2013-01-13T04:05:09+00:00"), :action => {:name => "/tweets/new"}).returns(events[1])
107
+ SkyDB::Event.expects(:new).with(:object_id => '67.228.32.10', :timestamp => DateTime.parse("2013-01-13T04:05:07+00:00"), :action => {:name => "/tweets"}).returns(events[2])
108
+ SkyDB::Event.expects(:new).with(:object_id => '157.55.35.85', :timestamp => DateTime.parse("2013-01-13T04:05:07+00:00"), :action => {:name => "/index.html"}).returns(events[3])
109
+ SkyDB::Event.expects(:new).with(:object_id => '66.29.187.16', :timestamp => DateTime.parse("2013-01-13T04:05:07+00:00"), :action => {:name => "/messages/1840832/open"}).returns(events[4])
110
+ SkyDB.expects(:add_event).with(events[0])
111
+ SkyDB.expects(:add_event).with(events[1])
112
+ SkyDB.expects(:add_event).with(events[2])
113
+ SkyDB.expects(:add_event).with(events[3])
114
+ SkyDB.expects(:add_event).with(events[4])
115
+ @importer.load_transform_file('apache')
116
+ @importer.import(['fixtures/unit/importer/simple.log'], :progress_bar => false)
117
+ end
118
+ assert_equal '', out
119
+ assert_equal '[import] Determining file type: apache_log', err.chomp
120
+ end
121
+
122
+ def test_import_bzip2_apache_log
123
+ out, err = capture_io do
124
+ event = mock()
125
+ SkyDB::Event.expects(:new).with(:object_id => '66.29.187.16', :timestamp => DateTime.parse("2013-01-13T04:05:07+00:00"), :action => {:name => "/users/1"}).returns(event)
126
+ SkyDB.expects(:add_event).with(event)
127
+ @importer.load_transform_file('apache')
128
+ @importer.file_type = :apache_log
129
+ @importer.import(['fixtures/unit/importer/simple.log.bz2'])
130
+ end
131
+ assert_equal '', out
132
+ assert_equal '', err
133
+ end
134
+
135
+ def test_import_gzip_apache_log
136
+ out, err = capture_io do
137
+ event = mock()
138
+ SkyDB::Event.expects(:new).with(:object_id => '66.29.187.16', :timestamp => DateTime.parse("2013-01-13T04:05:07+00:00"), :action => {:name => "/users/1"}).returns(event)
139
+ SkyDB.expects(:add_event).with(event)
140
+ @importer.load_transform_file('apache')
141
+ @importer.file_type = :apache_log
142
+ @importer.import(['fixtures/unit/importer/simple.log.gz'])
143
+ end
144
+ assert_equal '', out
145
+ assert_equal '', err
146
+ end
147
+
148
+ def test_import_override_file_type
149
+ out, err = capture_io do
150
+ event = mock()
151
+ SkyDB::Event.expects(:new).with(:object_id => '100', :timestamp => Chronic.parse("2000-01-01T00:00:00Z"), :action => {:name => '/index.html'}).returns(event)
152
+ SkyDB.expects(:add_event).with(event)
153
+ @importer.load_transform_file('sky')
154
+ @importer.file_type = :csv
155
+ @importer.import(['fixtures/unit/importer/simple'], :progress_bar => false)
156
+ end
157
+ assert_equal '', out
158
+ assert_equal '', err
159
+ end
160
+
161
+ def test_import_csv_no_headers
162
+ out, err = capture_io do
163
+ events = [mock(), mock(), mock(), mock()]
164
+ SkyDB::Event.expects(:new).with(:object_id => '100', :timestamp => Chronic.parse("2000-01-01T00:00:00Z"), :action => {:name => '/index.html'}).returns(events[0])
165
+ SkyDB.expects(:add_event).with(events[0])
166
+ @importer.headers = ['object_id', 'timestamp', 'action.name']
167
+ @importer.load_transform_file('sky')
168
+ @importer.import(['fixtures/unit/importer/no_headers.csv'], :progress_bar => false)
169
+ end
170
+ assert_equal '', out
171
+ assert_equal '[import] Determining file type: csv', err.chomp
172
+ end
173
+
174
+ def test_import_bad_timestamp
175
+ out, err = capture_io do
176
+ events = mock()
177
+ SkyDB::Event.expects(:new).never
178
+ SkyDB.expects(:add_event).never
179
+ @importer.load_transform_file('sky')
180
+ @importer.import(['fixtures/unit/importer/bad_timestamp.csv'], :progress_bar => false)
181
+ end
182
+ assert_equal '', out
183
+ assert_equal(
184
+ "[import] Determining file type: csv\n" +
185
+ "[ERROR] Invalid timestamp on line 2",
186
+ err.chomp
187
+ )
188
+ end
189
+
190
+ def test_import_unsupported_file_type
191
+ e = nil
192
+ out, err = capture_io do
193
+ e = assert_raises(SkyDB::Import::Importer::UnsupportedFileType) do
194
+ @importer.import(['fixtures/unit/importer/unsupported.xxx'], :progress_bar => false)
195
+ end
196
+ end
197
+ assert_equal 'File type not supported by importer: .xxx', e.message
198
+ assert_equal '', out
199
+ assert_equal "[import] Determining file type: ???", err.chomp
200
+ end
201
+
202
+ def test_import_transform_not_found
203
+ e = assert_raises(SkyDB::Import::Importer::TransformNotFound) do
204
+ @importer.load_transform_file('no_such_transform.yml')
205
+ end
206
+ assert_equal 'Transform file not found: no_such_transform.yml', e.message
207
+ end
208
+ end
File without changes
@@ -14,7 +14,7 @@ class TestMessageAddEvent < MiniTest::Unit::TestCase
14
14
  buffer = StringIO.new
15
15
  @message.table_name = "users"
16
16
  @message.event = SkyDB::Event.new(
17
- object_id: 12,
17
+ object_id: "12",
18
18
  timestamp:DateTime.parse('2010-01-02T10:30:20Z'),
19
19
  action: {
20
20
  name:"/index.html",
@@ -30,6 +30,6 @@ class TestMessageAddEvent < MiniTest::Unit::TestCase
30
30
  }
31
31
  )
32
32
  @message.encode(buffer)
33
- assert_bytes "\x93\x01\xa9add_event\xa5users\x84\xa8objectId\x0c\xa9timestamp\xcf\x00\x04\x7c\x2b\xf9\x9b\x87\x00\xa6action\x83\xa4name\xab\x2findex\x2ehtml\xa7astring\xa3foo\xa4aint\x14\xa4data\x85\xa7ostring\xa3bar\xa4oint\x0a\xa7odouble\xcb\x40Y\x06fffff\xa5otrue\xc3\xa6ofalse\xc2", buffer
33
+ assert_bytes "\x93\x01\xa9add_event\xa5users\x84\xa8objectId\xa3\xa212\xa9timestamp\xcf\x00\x04\x7c\x2b\xf9\x9b\x87\x00\xa6action\x83\xa4name\xab\x2findex\x2ehtml\xa7astring\xa3foo\xa4aint\x14\xa4data\x85\xa7ostring\xa3bar\xa4oint\x0a\xa7odouble\xcb\x40Y\x06fffff\xa5otrue\xc3\xa6ofalse\xc2", buffer
34
34
  end
35
35
  end
@@ -0,0 +1,18 @@
1
+ # encoding: binary
2
+ require 'test_helper'
3
+
4
+ class TestMessageGetTables < MiniTest::Unit::TestCase
5
+ def setup
6
+ @message = SkyDB::Message::GetTables.new()
7
+ end
8
+
9
+ ######################################
10
+ # Encoding
11
+ ######################################
12
+
13
+ def test_encode
14
+ buffer = StringIO.new
15
+ @message.encode(buffer)
16
+ assert_bytes "\x93\x01\xaaget_tables\xa0", buffer
17
+ end
18
+ end
File without changes
File without changes
@@ -0,0 +1,89 @@
1
+ require 'test_helper'
2
+
3
+ class TestQueryAfterCondition < MiniTest::Unit::TestCase
4
+ ##############################################################################
5
+ #
6
+ # Setup / Teardown
7
+ #
8
+ ##############################################################################
9
+
10
+ def setup
11
+ @after = SkyDB::Query::AfterCondition.new()
12
+ end
13
+
14
+
15
+ ##############################################################################
16
+ #
17
+ # Tests
18
+ #
19
+ ##############################################################################
20
+
21
+ ######################################
22
+ # Validation
23
+ ######################################
24
+
25
+ def test_validate_action
26
+ e = assert_raises(SkyDB::Query::ValidationError) do
27
+ SkyDB::Query::AfterCondition.new(:function_name => "foo").validate!
28
+ end
29
+ assert_match /^Action with non-zero identifier required/, e.message
30
+ end
31
+
32
+ def test_validate_enter_action
33
+ e = assert_raises(SkyDB::Query::ValidationError) do
34
+ SkyDB::Query::AfterCondition.new(:action => :enter, :function_name => "foo").validate!
35
+ end
36
+ assert_equal "Enter actions cannot be used with an 'after' condition. Please use an 'on' condition instead.", e.message
37
+ end
38
+
39
+ def test_validate_function_name
40
+ e = assert_raises(SkyDB::Query::ValidationError) do
41
+ SkyDB::Query::AfterCondition.new(:action => 10).validate!
42
+ end
43
+ assert_match /^Invalid function name ''/, e.message
44
+ end
45
+
46
+
47
+ ######################################
48
+ # Code Generation
49
+ ######################################
50
+
51
+ def test_codegen
52
+ @after = SkyDB::Query::AfterCondition.new(:action => 10, :function_name => "foo")
53
+ expected =
54
+ <<-BLOCK.unindent
55
+ function foo(cursor, data)
56
+ if cursor:eos() or cursor:eof() then return false end
57
+ repeat
58
+ if cursor.event.action_id == 10 then
59
+ cursor:next()
60
+ return true
61
+ end
62
+ until not cursor:next()
63
+ return false
64
+ end
65
+ BLOCK
66
+ assert_equal expected, @after.codegen()
67
+ end
68
+
69
+ def test_codegen_within_1_step
70
+ @after = SkyDB::Query::AfterCondition.new(:action => 10, :within => {:quantity => 1, :unit => 'step'}, :function_name => "foo")
71
+ expected =
72
+ <<-BLOCK.unindent
73
+ function foo(cursor, data)
74
+ if cursor:eos() or cursor:eof() then return false end
75
+ remaining = 1
76
+ repeat
77
+ if remaining <= 0 then return false end
78
+ if cursor.event.action_id == 10 then
79
+ cursor:next()
80
+ return true
81
+ end
82
+ remaining = remaining - 1
83
+ until not cursor:next()
84
+ return false
85
+ end
86
+ BLOCK
87
+ assert_equal expected, @after.codegen()
88
+ end
89
+ end
@@ -1,6 +1,6 @@
1
1
  require 'test_helper'
2
2
 
3
- class TestQueryAfter < MiniTest::Unit::TestCase
3
+ class TestQueryOnCondition < MiniTest::Unit::TestCase
4
4
  ##############################################################################
5
5
  #
6
6
  # Setup / Teardown
@@ -8,7 +8,7 @@ class TestQueryAfter < MiniTest::Unit::TestCase
8
8
  ##############################################################################
9
9
 
10
10
  def setup
11
- @after = SkyDB::Query::After.new()
11
+ @condition = SkyDB::Query::OnCondition.new()
12
12
  end
13
13
 
14
14
 
@@ -24,14 +24,14 @@ class TestQueryAfter < MiniTest::Unit::TestCase
24
24
 
25
25
  def test_validate_action
26
26
  e = assert_raises(SkyDB::Query::ValidationError) do
27
- SkyDB::Query::After.new(:function_name => "foo").validate!
27
+ SkyDB::Query::OnCondition.new(:function_name => "foo").validate!
28
28
  end
29
29
  assert_match /^Action with non-zero identifier required/, e.message
30
30
  end
31
-
31
+
32
32
  def test_validate_function_name
33
33
  e = assert_raises(SkyDB::Query::ValidationError) do
34
- SkyDB::Query::After.new(:action => 10).validate!
34
+ SkyDB::Query::OnCondition.new(:action => 10).validate!
35
35
  end
36
36
  assert_match /^Invalid function name ''/, e.message
37
37
  end
@@ -42,30 +42,30 @@ class TestQueryAfter < MiniTest::Unit::TestCase
42
42
  ######################################
43
43
 
44
44
  def test_codegen
45
- @after = SkyDB::Query::After.new(:action => 10, :function_name => "foo")
45
+ @condition = SkyDB::Query::OnCondition.new(:action => 10, :function_name => "foo")
46
46
  expected =
47
47
  <<-BLOCK.unindent
48
48
  function foo(cursor, data)
49
+ if cursor:eos() or cursor:eof() then return false end
49
50
  repeat
50
51
  if cursor.event.action_id == 10 then
51
- cursor:next()
52
52
  return true
53
53
  end
54
54
  until not cursor:next()
55
55
  return false
56
56
  end
57
57
  BLOCK
58
- assert_equal expected, @after.codegen()
58
+ assert_equal expected, @condition.codegen()
59
59
  end
60
60
 
61
61
  def test_codegen_enter
62
- @after = SkyDB::Query::After.new(:action => :enter, :function_name => "foo")
62
+ @condition = SkyDB::Query::OnCondition.new(:action => :enter, :function_name => "foo")
63
63
  expected =
64
64
  <<-BLOCK.unindent
65
65
  function foo(cursor, data)
66
66
  return (cursor.session_event_index == 0)
67
67
  end
68
68
  BLOCK
69
- assert_equal expected, @after.codegen()
69
+ assert_equal expected, @condition.codegen()
70
70
  end
71
71
  end