neptune 0.2.1 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. data/README +4 -0
  2. data/doc/BabelHelper.html +393 -376
  3. data/doc/BadConfigurationException.html +121 -127
  4. data/doc/CommonFunctions.html +237 -265
  5. data/doc/ExodusHelper.html +820 -0
  6. data/doc/ExodusTaskInfo.html +263 -0
  7. data/doc/FileNotFoundException.html +121 -127
  8. data/doc/NeptuneHelper.html +527 -592
  9. data/doc/NeptuneManagerClient.html +696 -0
  10. data/doc/NeptuneManagerException.html +139 -0
  11. data/doc/Object.html +334 -236
  12. data/doc/TaskInfo.html +428 -0
  13. data/doc/created.rid +8 -5
  14. data/doc/images/add.png +0 -0
  15. data/doc/images/delete.png +0 -0
  16. data/doc/images/tag_blue.png +0 -0
  17. data/doc/images/transparent.png +0 -0
  18. data/doc/index.html +74 -142
  19. data/doc/js/darkfish.js +99 -62
  20. data/doc/js/jquery.js +15 -29
  21. data/doc/js/navigation.js +142 -0
  22. data/doc/js/search.js +94 -0
  23. data/doc/js/search_index.js +1 -0
  24. data/doc/js/searcher.js +228 -0
  25. data/doc/table_of_contents.html +226 -0
  26. data/lib/babel.rb +116 -50
  27. data/lib/custom_exceptions.rb +2 -2
  28. data/lib/exodus.rb +311 -0
  29. data/lib/exodus_task_info.rb +36 -0
  30. data/lib/neptune.rb +52 -18
  31. data/lib/{app_controller_client.rb → neptune_manager_client.rb} +54 -38
  32. data/lib/task_info.rb +155 -0
  33. data/test/{unit/test_babel.rb → test_babel.rb} +161 -26
  34. data/test/{unit/test_common_functions.rb → test_common_functions.rb} +1 -1
  35. data/test/test_exodus.rb +687 -0
  36. data/test/{unit/test_neptune.rb → test_neptune.rb} +28 -17
  37. data/test/{unit/test_app_controller_client.rb → test_neptune_manager_client.rb} +15 -16
  38. data/test/test_task_info.rb +32 -0
  39. data/test/{unit/ts_all.rb → ts_all.rb} +3 -1
  40. metadata +30 -34
  41. data/doc/AppControllerClient.html +0 -702
  42. data/doc/AppControllerException.html +0 -145
  43. data/doc/bin/neptune.html +0 -56
  44. data/doc/js/quicksearch.js +0 -114
  45. data/doc/js/thickbox-compressed.js +0 -10
  46. data/doc/lib/app_controller_client_rb.html +0 -60
  47. data/doc/lib/babel_rb.html +0 -68
  48. data/doc/lib/common_functions_rb.html +0 -70
  49. data/doc/lib/custom_exceptions_rb.html +0 -54
  50. data/doc/lib/neptune_rb.html +0 -60
  51. data/test/integration/tc_c.rb +0 -57
  52. data/test/integration/tc_dfsp.rb +0 -37
  53. data/test/integration/tc_dwssa.rb +0 -38
  54. data/test/integration/tc_erlang.rb +0 -183
  55. data/test/integration/tc_mapreduce.rb +0 -282
  56. data/test/integration/tc_mpi.rb +0 -160
  57. data/test/integration/tc_storage.rb +0 -209
  58. data/test/integration/tc_upc.rb +0 -75
  59. data/test/integration/tc_x10.rb +0 -94
  60. data/test/integration/test_helper.rb +0 -135
  61. data/test/integration/ts_neptune.rb +0 -40
@@ -1,75 +0,0 @@
1
-
2
- class TestUPC < Test::Unit::TestCase
3
- def test_hello_world_code
4
- STORAGE_TYPES.each { |storage|
5
- run_hello_world_code(storage)
6
- }
7
- end
8
-
9
- def run_hello_world_code(storage)
10
- expected_output = "Hello from thread 0"
11
- ring_code = <<BAZ
12
- #include <upc_relaxed.h>
13
- #include <stdio.h>
14
-
15
- int main() {
16
- printf("Hello from thread %i/%i", MYTHREAD, THREADS);
17
- upc_barrier;
18
- return 0;
19
- }
20
-
21
- BAZ
22
-
23
- contents = TestHelper.get_random_alphanumeric(1024)
24
- folder = "hello-world-#{TestHelper.get_random_alphanumeric}"
25
- source = "HelloWorld.c"
26
-
27
- tmp_folder = "/tmp/#{folder}"
28
- FileUtils.mkdir_p(tmp_folder)
29
- compiled = "#{tmp_folder}-compiled"
30
- compiled_code = "#{compiled}/HelloWorld"
31
-
32
- local = "#{tmp_folder}/#{source}"
33
- TestHelper.write_file(local, ring_code)
34
-
35
- output = TestHelper.get_output_location(folder, storage)
36
-
37
- compile_upc_code(tmp_folder, source, compiled)
38
- start_upc_code(compiled_code, output, storage)
39
- get_upc_output(output, expected_output, storage)
40
-
41
- FileUtils.rm_rf(tmp_folder)
42
- FileUtils.rm_rf(compiled)
43
- end
44
-
45
- def compile_upc_code(location, main_file, compiled)
46
- std_out, std_err = TestHelper.compile_code(location, main_file, compiled)
47
-
48
- make = "/usr/local/berkeley_upc-2.12.1/upcc --network=mpi -o HelloWorld HelloWorld.c"
49
- msg = "The UPC code did not compile as expected. It should have " +
50
- "compiled with the command [#{make}] instead of [#{std_out}]."
51
- assert_equal(std_out, make, msg)
52
-
53
- msg = "The UPC code did not compile successfully. It reported " +
54
- "the following error: #{std_err}"
55
- assert_nil(std_err, msg)
56
- end
57
-
58
- def start_upc_code(code_location, output, storage)
59
- status = TestHelper.start_job("upc", code_location, output, storage)
60
-
61
- msg = "Your job was not started successfully. The failure message " +
62
- "reported was #{status[:msg]}"
63
- assert_equal(status[:result], :success, msg)
64
- end
65
-
66
- def get_upc_output(output, expected, storage)
67
- result = TestHelper.get_job_output(output, storage)
68
-
69
- msg = "The UPC job you ran did not return the expected result. " +
70
- "We expected to see [#{expected}] but instead saw [#{result}]"
71
- success = result.include?(expected)
72
- assert(success, msg)
73
- end
74
- end
75
-
@@ -1,94 +0,0 @@
1
-
2
- class TestX10 < Test::Unit::TestCase
3
- def test_ring_code
4
- STORAGE_TYPES.each { |storage|
5
- run_ring_code(storage)
6
- }
7
- end
8
-
9
- def run_ring_code(storage)
10
- expected_output = "All done!"
11
- ring_code = <<BAZ
12
- import x10.lang.Math;
13
- import x10.util.Timer;
14
-
15
- public class Ring {
16
-
17
- static val NUM_MESSAGES = 1;
18
-
19
- // A global datastructure with one integer cell per place
20
- static A = PlaceLocalHandle.make[Cell[Long]](Dist.makeUnique(), ()=>new Cell[Long](-1));
21
-
22
- public static def send (msg:Long, depth:Int) {
23
- A()() = msg;
24
- if (depth==0) return;
25
- async at (here.next()) send(msg, depth-1);
26
- }
27
-
28
- public static def main(args:Array[String](1)) {
29
-
30
- val startTime = Timer.milliTime();
31
- finish send(42L, NUM_MESSAGES * Place.MAX_PLACES);
32
- val endTime = Timer.milliTime();
33
-
34
- val totalTime = (endTime - startTime) / 1000.0;
35
-
36
- Console.OUT.printf("#{expected_output}");
37
- }
38
- }
39
-
40
- BAZ
41
-
42
- contents = TestHelper.get_random_alphanumeric(1024)
43
- folder = "ring-#{TestHelper.get_random_alphanumeric}"
44
- source = "Ring.x10"
45
-
46
- tmp_folder = "/tmp/#{folder}"
47
- FileUtils.mkdir_p(tmp_folder)
48
- compiled = "#{tmp_folder}-compiled"
49
- compiled_code = "#{compiled}/Ring"
50
-
51
- local = "#{tmp_folder}/#{source}"
52
- TestHelper.write_file(local, ring_code)
53
-
54
- output = TestHelper.get_output_location(folder, storage)
55
-
56
- compile_x10_code(tmp_folder, source, compiled)
57
- start_x10_code(compiled_code, output, storage)
58
- get_x10_output(output, expected_output, storage)
59
-
60
- FileUtils.rm_rf(tmp_folder)
61
- FileUtils.rm_rf(compiled)
62
- end
63
-
64
- def compile_x10_code(location, main_file, compiled)
65
- std_out, std_err = TestHelper.compile_code(location, main_file, compiled)
66
-
67
- make = "/usr/local/x10/x10.dist/bin/x10c++ -x10rt mpi -o Ring Ring.x10"
68
- msg = "The X10 Ring code did not compile as expected. It should have " +
69
- "compiled with the command [#{make}] instead of [#{std_out}]."
70
- assert_equal(std_out, make, msg)
71
-
72
- msg = "The X10 Ring code did not compile successfully. It reported " +
73
- "the following error: #{std_err}"
74
- compile_success = !std_err.include?("error")
75
- assert(compile_success, msg)
76
- end
77
-
78
- def start_x10_code(code_location, output, storage)
79
- status = TestHelper.start_job("x10", code_location, output, storage)
80
-
81
- msg = "Your job was not started successfully. The failure message " +
82
- "reported was #{status[:msg]}"
83
- assert_equal(status[:result], :success, msg)
84
- end
85
-
86
- def get_x10_output(output, expected, storage)
87
- result = TestHelper.get_job_output(output, storage)
88
-
89
- msg = "The X10 job you ran did not return the expected result. " +
90
- "We expected to see [#{expected}] but instead saw [#{result}]"
91
- assert_equal(result, expected, msg)
92
- end
93
- end
94
-
@@ -1,135 +0,0 @@
1
- #!/usr/bin/ruby -w
2
- # Programmer: Chris Bunch
3
-
4
- module TestHelper
5
- def self.compile_code(location, main_file, compiled_location)
6
- result = neptune(
7
- :type => "compile",
8
- :code => location,
9
- :main => main_file,
10
- :output => "/baz",
11
- :copy_to => compiled_location
12
- )
13
-
14
- puts "standard out is #{result[:out]}"
15
- puts "standard err is #{result[:err]}"
16
-
17
- return result[:out], result[:err]
18
- end
19
-
20
- def self.start_job(type, code_location, output, storage, extras={})
21
- params = {
22
- :type => type,
23
- :code => code_location,
24
- :output => output,
25
- :nodes_to_use => 1
26
- }.merge(TestHelper.get_storage_params(storage)).merge(extras)
27
-
28
- status = nil
29
-
30
- loop {
31
- status = neptune(params)
32
- if status[:msg] =~ /not enough free nodes/
33
- puts status[:msg]
34
- else
35
- break
36
- end
37
- sleep(5)
38
- }
39
-
40
- return status
41
- end
42
-
43
- def self.get_job_output(output, storage)
44
- result = ""
45
-
46
- params = {
47
- :type => "output",
48
- :output => output
49
- }.merge(TestHelper.get_storage_params(storage))
50
-
51
- loop {
52
- result = neptune(params)
53
-
54
- break if result != "error: output does not exist"
55
- puts "Waiting for job to complete..."
56
- sleep(30)
57
- }
58
-
59
- return result
60
- end
61
-
62
- def self.get_output_location(file, storage="appdb", notxt=false)
63
- output = "/neptune"
64
-
65
- if storage == "walrus"
66
- output << "_"
67
- else
68
- output << "-"
69
- end
70
-
71
- output << "testbin/#{file}"
72
- if !notxt
73
- output << ".txt"
74
- end
75
-
76
- return output
77
- end
78
-
79
- def self.get_storage_params(storage)
80
- if storage == "gstorage"
81
- return {
82
- :storage => "gstorage",
83
- :EC2_ACCESS_KEY => ENV['GSTORAGE_ACCESS_KEY'],
84
- :EC2_SECRET_KEY => ENV['GSTORAGE_SECRET_KEY'],
85
- :S3_URL => ENV['GSTORAGE_URL']
86
- }
87
- elsif storage == "s3"
88
- return {
89
- :storage => "s3",
90
- :EC2_ACCESS_KEY => ENV['S3_ACCESS_KEY'],
91
- :EC2_SECRET_KEY => ENV['S3_SECRET_KEY'],
92
- :S3_URL => ENV['S3_URL']
93
- }
94
- elsif storage == "walrus"
95
- return {
96
- :storage => "s3",
97
- :EC2_ACCESS_KEY => ENV['WALRUS_ACCESS_KEY'],
98
- :EC2_SECRET_KEY => ENV['WALRUS_SECRET_KEY'],
99
- :S3_URL => ENV['WALRUS_URL']
100
- }
101
- else
102
- return { :storage => storage }
103
- # nothing special to do
104
- end
105
- end
106
-
107
- def self.write_file(location, contents)
108
- File.open(location, "w+") { |file| file.write(contents) }
109
- end
110
-
111
- def self.read_file(location)
112
- File.open(location) { |f| f.read.chomp! }
113
- end
114
-
115
- def self.get_random_alphanumeric(length=10)
116
- random = ""
117
- possible = "0123456789abcdefghijklmnopqrstuvxwyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
118
- possibleLength = possible.length
119
-
120
- length.times { |index|
121
- random << possible[rand(possibleLength)]
122
- }
123
-
124
- return random
125
- end
126
-
127
- def self.is_appscale_running?(ip)
128
- begin
129
- Net::HTTP.get_response(URI.parse("http://#{ip}"))
130
- return true
131
- rescue Exception
132
- return false
133
- end
134
- end
135
- end
@@ -1,40 +0,0 @@
1
- STORAGE_TYPES = ["appdb", "gstorage", "s3", "walrus"] - ["appdb"]
2
-
3
- $:.unshift File.join(File.dirname(__FILE__), "..", "..", "lib")
4
- require 'neptune'
5
-
6
- $:.unshift File.join(File.dirname(__FILE__), "..", "test", "integration")
7
- require 'test_helper'
8
-
9
- REQUIRED_CREDS = %w{ APPSCALE_HEAD_NODE
10
- GSTORAGE_ACCESS_KEY GSTORAGE_SECRET_KEY GSTORAGE_URL
11
- S3_ACCESS_KEY S3_SECRET_KEY S3_URL
12
- WALRUS_ACCESS_KEY WALRUS_SECRET_KEY WALRUS_URL }
13
-
14
- require 'test/unit'
15
- require 'rubygems'
16
- require 'flexmock/test_unit'
17
-
18
- REQUIRED_CREDS.each { |cred|
19
- msg = "The environment variable #{cred} was not set. Please " +
20
- "set it and try again."
21
- abort(msg) if ENV[cred].nil?
22
- }
23
-
24
- APPSCALE_HEAD_NODE_IP = ENV['APPSCALE_HEAD_NODE']
25
- msg = "AppScale is not currently running at " +
26
- "#{APPSCALE_HEAD_NODE_IP}. Please start AppScale and try again."
27
- abort(msg) unless TestHelper.is_appscale_running?(APPSCALE_HEAD_NODE_IP)
28
-
29
- # TODO: refactor dfsp and dwssa to use the new ssa job type
30
-
31
- require 'tc_c'
32
- #require 'tc_dfsp'
33
- #require 'tc_dwssa'
34
- require 'tc_erlang'
35
- require 'tc_mapreduce'
36
- require 'tc_mpi'
37
- require 'tc_storage'
38
- require 'tc_upc'
39
- require 'tc_x10'
40
-