ganapati 0.0.4 → 0.0.5

Sign up to get free protection for your applications and to get access to all the features.
Files changed (76) hide show
  1. data/LICENSE +674 -0
  2. data/README.rdoc +86 -0
  3. data/Rakefile +32 -0
  4. data/bin/hcp +64 -0
  5. data/bin/hls +42 -0
  6. data/docs/classes/Ganapati.html +138 -0
  7. data/docs/classes/Ganapati/Client.html +666 -0
  8. data/docs/classes/Ganapati/FileUrl.html +253 -0
  9. data/docs/classes/Ganapati/HFile.html +271 -0
  10. data/docs/classes/ThriftHadoopFileSystem.html +169 -0
  11. data/docs/classes/ThriftHadoopFileSystem/Append_args.html +193 -0
  12. data/docs/classes/ThriftHadoopFileSystem/Append_result.html +198 -0
  13. data/docs/classes/ThriftHadoopFileSystem/BlockLocation.html +208 -0
  14. data/docs/classes/ThriftHadoopFileSystem/Chmod_args.html +198 -0
  15. data/docs/classes/ThriftHadoopFileSystem/Chmod_result.html +193 -0
  16. data/docs/classes/ThriftHadoopFileSystem/Chown_args.html +203 -0
  17. data/docs/classes/ThriftHadoopFileSystem/Chown_result.html +193 -0
  18. data/docs/classes/ThriftHadoopFileSystem/Client.html +1566 -0
  19. data/docs/classes/ThriftHadoopFileSystem/Close_args.html +193 -0
  20. data/docs/classes/ThriftHadoopFileSystem/Close_result.html +198 -0
  21. data/docs/classes/ThriftHadoopFileSystem/CreateFile_args.html +218 -0
  22. data/docs/classes/ThriftHadoopFileSystem/CreateFile_result.html +198 -0
  23. data/docs/classes/ThriftHadoopFileSystem/Create_args.html +193 -0
  24. data/docs/classes/ThriftHadoopFileSystem/Create_result.html +198 -0
  25. data/docs/classes/ThriftHadoopFileSystem/Exists_args.html +193 -0
  26. data/docs/classes/ThriftHadoopFileSystem/Exists_result.html +198 -0
  27. data/docs/classes/ThriftHadoopFileSystem/FileStatus.html +233 -0
  28. data/docs/classes/ThriftHadoopFileSystem/GetFileBlockLocations_args.html +203 -0
  29. data/docs/classes/ThriftHadoopFileSystem/GetFileBlockLocations_result.html +198 -0
  30. data/docs/classes/ThriftHadoopFileSystem/ListStatus_args.html +193 -0
  31. data/docs/classes/ThriftHadoopFileSystem/ListStatus_result.html +198 -0
  32. data/docs/classes/ThriftHadoopFileSystem/MalformedInputException.html +220 -0
  33. data/docs/classes/ThriftHadoopFileSystem/Mkdirs_args.html +193 -0
  34. data/docs/classes/ThriftHadoopFileSystem/Mkdirs_result.html +198 -0
  35. data/docs/classes/ThriftHadoopFileSystem/Open_args.html +193 -0
  36. data/docs/classes/ThriftHadoopFileSystem/Open_result.html +198 -0
  37. data/docs/classes/ThriftHadoopFileSystem/Pathname.html +193 -0
  38. data/docs/classes/ThriftHadoopFileSystem/Processor.html +710 -0
  39. data/docs/classes/ThriftHadoopFileSystem/Read_args.html +203 -0
  40. data/docs/classes/ThriftHadoopFileSystem/Read_result.html +198 -0
  41. data/docs/classes/ThriftHadoopFileSystem/Rename_args.html +198 -0
  42. data/docs/classes/ThriftHadoopFileSystem/Rename_result.html +198 -0
  43. data/docs/classes/ThriftHadoopFileSystem/Rm_args.html +198 -0
  44. data/docs/classes/ThriftHadoopFileSystem/Rm_result.html +198 -0
  45. data/docs/classes/ThriftHadoopFileSystem/SetInactivityTimeoutPeriod_args.html +199 -0
  46. data/docs/classes/ThriftHadoopFileSystem/SetInactivityTimeoutPeriod_result.html +188 -0
  47. data/docs/classes/ThriftHadoopFileSystem/SetReplication_args.html +198 -0
  48. data/docs/classes/ThriftHadoopFileSystem/SetReplication_result.html +193 -0
  49. data/docs/classes/ThriftHadoopFileSystem/Shutdown_args.html +193 -0
  50. data/docs/classes/ThriftHadoopFileSystem/Shutdown_result.html +188 -0
  51. data/docs/classes/ThriftHadoopFileSystem/Stat_args.html +193 -0
  52. data/docs/classes/ThriftHadoopFileSystem/Stat_result.html +198 -0
  53. data/docs/classes/ThriftHadoopFileSystem/ThriftHandle.html +193 -0
  54. data/docs/classes/ThriftHadoopFileSystem/ThriftIOException.html +220 -0
  55. data/docs/classes/ThriftHadoopFileSystem/Write_args.html +198 -0
  56. data/docs/classes/ThriftHadoopFileSystem/Write_result.html +198 -0
  57. data/docs/created.rid +1 -0
  58. data/docs/files/README_rdoc.html +209 -0
  59. data/docs/files/lib/ganapati/client_rb.html +101 -0
  60. data/docs/files/lib/ganapati/hfile_rb.html +101 -0
  61. data/docs/files/lib/ganapati/utils_rb.html +101 -0
  62. data/docs/files/lib/ganapati/version_rb.html +101 -0
  63. data/docs/files/lib/ganapati_rb.html +112 -0
  64. data/docs/files/lib/thrift/hadoopfs_constants_rb.html +117 -0
  65. data/docs/files/lib/thrift/hadoopfs_types_rb.html +110 -0
  66. data/docs/files/lib/thrift/thrift_hadoop_file_system_rb.html +118 -0
  67. data/docs/fr_class_index.html +77 -0
  68. data/docs/fr_file_index.html +35 -0
  69. data/docs/fr_method_index.html +222 -0
  70. data/docs/index.html +24 -0
  71. data/docs/rdoc-style.css +208 -0
  72. data/lib/ganapati.rb +1 -0
  73. data/lib/ganapati/client.rb +4 -2
  74. data/lib/ganapati/utils.rb +47 -0
  75. data/lib/ganapati/version.rb +3 -0
  76. metadata +79 -4
data/docs/created.rid ADDED
@@ -0,0 +1 @@
1
+ Thu, 31 Mar 2011 11:17:55 -0400
data/docs/files/README_rdoc.html ADDED
@@ -0,0 +1,209 @@
1
+ <?xml version="1.0" encoding="iso-8859-1"?>
2
+ <!DOCTYPE html
3
+ PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
4
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
5
+
6
+ <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
7
+ <head>
8
+ <title>File: README.rdoc</title>
9
+ <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
10
+ <meta http-equiv="Content-Script-Type" content="text/javascript" />
11
+ <link rel="stylesheet" href=".././rdoc-style.css" type="text/css" media="screen" />
12
+ <script type="text/javascript">
13
+ // <![CDATA[
14
+
15
+ function popupCode( url ) {
16
+ window.open(url, "Code", "resizable=yes,scrollbars=yes,toolbar=no,status=no,height=150,width=400")
17
+ }
18
+
19
+ function toggleCode( id ) {
20
+ if ( document.getElementById )
21
+ elem = document.getElementById( id );
22
+ else if ( document.all )
23
+ elem = eval( "document.all." + id );
24
+ else
25
+ return false;
26
+
27
+ elemStyle = elem.style;
28
+
29
+ if ( elemStyle.display != "block" ) {
30
+ elemStyle.display = "block"
31
+ } else {
32
+ elemStyle.display = "none"
33
+ }
34
+
35
+ return true;
36
+ }
37
+
38
+ // Make codeblocks hidden by default
39
+ document.writeln( "<style type=\"text/css\">div.method-source-code { display: none }</style>" )
40
+
41
+ // ]]>
42
+ </script>
43
+
44
+ </head>
45
+ <body>
46
+
47
+
48
+
49
+ <div id="fileHeader">
50
+ <h1>README.rdoc</h1>
51
+ <table class="header-table">
52
+ <tr class="top-aligned-row">
53
+ <td><strong>Path:</strong></td>
54
+ <td>README.rdoc
55
+ </td>
56
+ </tr>
57
+ <tr class="top-aligned-row">
58
+ <td><strong>Last Update:</strong></td>
59
+ <td>Thu Mar 31 11:09:15 -0400 2011</td>
60
+ </tr>
61
+ </table>
62
+ </div>
63
+ <!-- banner header -->
64
+
65
+ <div id="bodyContent">
66
+
67
+
68
+
69
+ <div id="contextContent">
70
+
71
+ <div id="description">
72
+ <h1>ganapati &#8212; Hadoop HDFS Thrift interface for Ruby</h1>
73
+ <p>
74
+ <a href="../classes/Ganapati.html">Ganapati</a> is a Ruby thrift lib for
75
+ interfacing with Hadoop&#8216;s distributed file system, HDFS. It also
76
+ includes a few command line client utilities.
77
+ </p>
78
+ <p>
79
+ To install:
80
+ </p>
81
+ <pre>
82
+ gem install ganapati
83
+ </pre>
84
+ <h2>Starting thrift server</h2>
85
+ <p>
86
+ Documentation in Hadoop for the thrift interface to HDFS is crap. It can be
87
+ found <a href="http://wiki.apache.org/hadoop/HDFS-APIs">here</a>.
88
+ </p>
89
+ <p>
90
+ As a much simpler and safer way of auto compiling and then starting the
91
+ thrift interface, use the provided script:
92
+ </p>
93
+ <pre>
94
+ bin/hdfs_thrift_server &lt;port&gt;
95
+ </pre>
96
+ <p>
97
+ This will start a thrift server on the given port (after compiling the
98
+ server code provided in the Hadoop distribution).
99
+ </p>
100
+ <h2>Basic Usage</h2>
101
+ <pre>
102
+ require 'rubygems'
103
+ require 'ganapati'
104
+
105
+ # args are host, port, and optional timeout
106
+ client = Ganapati::Client.new 'localhost', 1234
107
+
108
+ # copy a file to hdfs
109
+ client.put(&quot;/some/file&quot;, &quot;/some/hadoop/path&quot;)
110
+
111
+ # get a file from hadoop
112
+ client.get(&quot;/some/hadoop/path&quot;, &quot;/local/path&quot;)
113
+
114
+ # Create a file
115
+ f = client.create(&quot;/home/someuser/afile.txt&quot;)
116
+ f.write(&quot;this is some text&quot;)
117
+ # Always, always close the file
118
+ f.close
119
+
120
+ # Create a file with code block
121
+ client.create(&quot;/home/someuser/afile.txt&quot;) { |f|
122
+ f.write(&quot;this is some text&quot;)
123
+ }
124
+
125
+ # Open a file for reading and read it
126
+ client.open('/home/someuser/afile.txt') { |f|
127
+ puts f.read
128
+ # or read for specific length from start
129
+ puts f.read(0, 4)
130
+ }
131
+
132
+ # read a file line by line
133
+ client.readlines('/home/someuser/afile.txt') { |line|
134
+ puts line
135
+ }
136
+
137
+ # Open a file for appending and append to it
138
+ client.append('/home/someuser/afile.txt') { |f|
139
+ f.write &quot;new data&quot;
140
+ }
141
+
142
+ ## Common file methods are available (chown, chmod, mkdir, stat, etc). Examples:
143
+ # move a file
144
+ client.mv &quot;/home/someuser/afile.txt&quot;, &quot;/home/someuser/test.txt&quot;
145
+
146
+ # remove a file
147
+ client.rm &quot;/home/someuser/test.txt&quot;
148
+
149
+ # test for file existance
150
+ client.exists? &quot;/home/someuser/test.txt&quot;
151
+
152
+ # get a list of all files
153
+ client.ls &quot;/home&quot;
154
+
155
+ client.close
156
+
157
+ # Quick and dirty way to print remote file. The run class method takes care of closing the client.
158
+ puts Ganapati::Client.run('localhost', 1234) { |c| c.open('/home/someuser/afile.txt') { |f| f.read } }
159
+ </pre>
160
+ <h2>Command Line Utilities</h2>
161
+ <p>
162
+ There are a few utility programs included in the bin directory. <b>hls</b>
163
+ provides a way to see the contents of HDFS (recursively and verbosely with
164
+ appropriate command line options):
165
+ </p>
166
+ <pre>
167
+ ./bin/hls hdfs://host:port/tmp
168
+ </pre>
169
+ <p>
170
+ <b>hcp</b> provides a way to copy to/from/between HDFS servers:
171
+ </p>
172
+ <pre>
173
+ ./bin/hcp hdfs://host:port/some/path/to/file ./file
174
+ ./bin/hcp ./file hdfs://host:port/some/path/to/file
175
+ ./bin/hcp hdfs://anotherhost:port/some/path/to/file hdfs://host:port/some/path/to/file
176
+ </pre>
177
+
178
+ </div>
179
+
180
+
181
+ </div>
182
+
183
+
184
+ </div>
185
+
186
+
187
+ <!-- if includes -->
188
+
189
+ <div id="section">
190
+
191
+
192
+
193
+
194
+
195
+
196
+
197
+
198
+ <!-- if method_list -->
199
+
200
+
201
+ </div>
202
+
203
+
204
+ <div id="validator-badges">
205
+ <p><small><a href="http://validator.w3.org/check/referer">[Validate]</a></small></p>
206
+ </div>
207
+
208
+ </body>
209
+ </html>
data/docs/files/lib/ganapati/client_rb.html ADDED
@@ -0,0 +1,101 @@
1
+ <?xml version="1.0" encoding="iso-8859-1"?>
2
+ <!DOCTYPE html
3
+ PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
4
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
5
+
6
+ <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
7
+ <head>
8
+ <title>File: client.rb</title>
9
+ <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
10
+ <meta http-equiv="Content-Script-Type" content="text/javascript" />
11
+ <link rel="stylesheet" href="../../.././rdoc-style.css" type="text/css" media="screen" />
12
+ <script type="text/javascript">
13
+ // <![CDATA[
14
+
15
+ function popupCode( url ) {
16
+ window.open(url, "Code", "resizable=yes,scrollbars=yes,toolbar=no,status=no,height=150,width=400")
17
+ }
18
+
19
+ function toggleCode( id ) {
20
+ if ( document.getElementById )
21
+ elem = document.getElementById( id );
22
+ else if ( document.all )
23
+ elem = eval( "document.all." + id );
24
+ else
25
+ return false;
26
+
27
+ elemStyle = elem.style;
28
+
29
+ if ( elemStyle.display != "block" ) {
30
+ elemStyle.display = "block"
31
+ } else {
32
+ elemStyle.display = "none"
33
+ }
34
+
35
+ return true;
36
+ }
37
+
38
+ // Make codeblocks hidden by default
39
+ document.writeln( "<style type=\"text/css\">div.method-source-code { display: none }</style>" )
40
+
41
+ // ]]>
42
+ </script>
43
+
44
+ </head>
45
+ <body>
46
+
47
+
48
+
49
+ <div id="fileHeader">
50
+ <h1>client.rb</h1>
51
+ <table class="header-table">
52
+ <tr class="top-aligned-row">
53
+ <td><strong>Path:</strong></td>
54
+ <td>lib/ganapati/client.rb
55
+ </td>
56
+ </tr>
57
+ <tr class="top-aligned-row">
58
+ <td><strong>Last Update:</strong></td>
59
+ <td>Thu Mar 31 10:26:28 -0400 2011</td>
60
+ </tr>
61
+ </table>
62
+ </div>
63
+ <!-- banner header -->
64
+
65
+ <div id="bodyContent">
66
+
67
+
68
+
69
+ <div id="contextContent">
70
+
71
+
72
+
73
+ </div>
74
+
75
+
76
+ </div>
77
+
78
+
79
+ <!-- if includes -->
80
+
81
+ <div id="section">
82
+
83
+
84
+
85
+
86
+
87
+
88
+
89
+
90
+ <!-- if method_list -->
91
+
92
+
93
+ </div>
94
+
95
+
96
+ <div id="validator-badges">
97
+ <p><small><a href="http://validator.w3.org/check/referer">[Validate]</a></small></p>
98
+ </div>
99
+
100
+ </body>
101
+ </html>
data/docs/files/lib/ganapati/hfile_rb.html ADDED
@@ -0,0 +1,101 @@
1
+ <?xml version="1.0" encoding="iso-8859-1"?>
2
+ <!DOCTYPE html
3
+ PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
4
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
5
+
6
+ <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
7
+ <head>
8
+ <title>File: hfile.rb</title>
9
+ <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
10
+ <meta http-equiv="Content-Script-Type" content="text/javascript" />
11
+ <link rel="stylesheet" href="../../.././rdoc-style.css" type="text/css" media="screen" />
12
+ <script type="text/javascript">
13
+ // <![CDATA[
14
+
15
+ function popupCode( url ) {
16
+ window.open(url, "Code", "resizable=yes,scrollbars=yes,toolbar=no,status=no,height=150,width=400")
17
+ }
18
+
19
+ function toggleCode( id ) {
20
+ if ( document.getElementById )
21
+ elem = document.getElementById( id );
22
+ else if ( document.all )
23
+ elem = eval( "document.all." + id );
24
+ else
25
+ return false;
26
+
27
+ elemStyle = elem.style;
28
+
29
+ if ( elemStyle.display != "block" ) {
30
+ elemStyle.display = "block"
31
+ } else {
32
+ elemStyle.display = "none"
33
+ }
34
+
35
+ return true;
36
+ }
37
+
38
+ // Make codeblocks hidden by default
39
+ document.writeln( "<style type=\"text/css\">div.method-source-code { display: none }</style>" )
40
+
41
+ // ]]>
42
+ </script>
43
+
44
+ </head>
45
+ <body>
46
+
47
+
48
+
49
+ <div id="fileHeader">
50
+ <h1>hfile.rb</h1>
51
+ <table class="header-table">
52
+ <tr class="top-aligned-row">
53
+ <td><strong>Path:</strong></td>
54
+ <td>lib/ganapati/hfile.rb
55
+ </td>
56
+ </tr>
57
+ <tr class="top-aligned-row">
58
+ <td><strong>Last Update:</strong></td>
59
+ <td>Wed Jan 19 14:29:41 -0500 2011</td>
60
+ </tr>
61
+ </table>
62
+ </div>
63
+ <!-- banner header -->
64
+
65
+ <div id="bodyContent">
66
+
67
+
68
+
69
+ <div id="contextContent">
70
+
71
+
72
+
73
+ </div>
74
+
75
+
76
+ </div>
77
+
78
+
79
+ <!-- if includes -->
80
+
81
+ <div id="section">
82
+
83
+
84
+
85
+
86
+
87
+
88
+
89
+
90
+ <!-- if method_list -->
91
+
92
+
93
+ </div>
94
+
95
+
96
+ <div id="validator-badges">
97
+ <p><small><a href="http://validator.w3.org/check/referer">[Validate]</a></small></p>
98
+ </div>
99
+
100
+ </body>
101
+ </html>
data/docs/files/lib/ganapati/utils_rb.html ADDED
@@ -0,0 +1,101 @@
1
+ <?xml version="1.0" encoding="iso-8859-1"?>
2
+ <!DOCTYPE html
3
+ PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
4
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
5
+
6
+ <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
7
+ <head>
8
+ <title>File: utils.rb</title>
9
+ <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
10
+ <meta http-equiv="Content-Script-Type" content="text/javascript" />
11
+ <link rel="stylesheet" href="../../.././rdoc-style.css" type="text/css" media="screen" />
12
+ <script type="text/javascript">
13
+ // <![CDATA[
14
+
15
+ function popupCode( url ) {
16
+ window.open(url, "Code", "resizable=yes,scrollbars=yes,toolbar=no,status=no,height=150,width=400")
17
+ }
18
+
19
+ function toggleCode( id ) {
20
+ if ( document.getElementById )
21
+ elem = document.getElementById( id );
22
+ else if ( document.all )
23
+ elem = eval( "document.all." + id );
24
+ else
25
+ return false;
26
+
27
+ elemStyle = elem.style;
28
+
29
+ if ( elemStyle.display != "block" ) {
30
+ elemStyle.display = "block"
31
+ } else {
32
+ elemStyle.display = "none"
33
+ }
34
+
35
+ return true;
36
+ }
37
+
38
+ // Make codeblocks hidden by default
39
+ document.writeln( "<style type=\"text/css\">div.method-source-code { display: none }</style>" )
40
+
41
+ // ]]>
42
+ </script>
43
+
44
+ </head>
45
+ <body>
46
+
47
+
48
+
49
+ <div id="fileHeader">
50
+ <h1>utils.rb</h1>
51
+ <table class="header-table">
52
+ <tr class="top-aligned-row">
53
+ <td><strong>Path:</strong></td>
54
+ <td>lib/ganapati/utils.rb
55
+ </td>
56
+ </tr>
57
+ <tr class="top-aligned-row">
58
+ <td><strong>Last Update:</strong></td>
59
+ <td>Thu Mar 31 10:31:24 -0400 2011</td>
60
+ </tr>
61
+ </table>
62
+ </div>
63
+ <!-- banner header -->
64
+
65
+ <div id="bodyContent">
66
+
67
+
68
+
69
+ <div id="contextContent">
70
+
71
+
72
+
73
+ </div>
74
+
75
+
76
+ </div>
77
+
78
+
79
+ <!-- if includes -->
80
+
81
+ <div id="section">
82
+
83
+
84
+
85
+
86
+
87
+
88
+
89
+
90
+ <!-- if method_list -->
91
+
92
+
93
+ </div>
94
+
95
+
96
+ <div id="validator-badges">
97
+ <p><small><a href="http://validator.w3.org/check/referer">[Validate]</a></small></p>
98
+ </div>
99
+
100
+ </body>
101
+ </html>