swineherd 0.0.4 → 0.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/VERSION +1 -1
- data/lib/swineherd/filesystem/hadoopfilesystem.rb +4 -2
- data/lib/swineherd/script.rb +7 -11
- data/swineherd.gemspec +4 -5
- data/tests/testcfg.yaml +2 -2
- metadata +8 -41
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.0.
|
1
|
+
0.0.6
|
@@ -14,10 +14,12 @@ module Swineherd
|
|
14
14
|
#
|
15
15
|
# Initialize a new hadoop file system, needs path to hadoop configuration
|
16
16
|
#
|
17
|
-
def initialize *args
|
17
|
+
def initialize params={}, *args
|
18
18
|
check_and_set_environment
|
19
19
|
@conf = Java::org.apache.hadoop.conf.Configuration.new
|
20
|
-
|
20
|
+
uri = Java::java.net.URI.new params[:filesystem] if params[:filesystem]
|
21
|
+
fs_params = uri ? [ uri, @conf ] : [ @conf ]
|
22
|
+
@hdfs = Java::org.apache.hadoop.fs.FileSystem.get *fs_params
|
21
23
|
end
|
22
24
|
|
23
25
|
#
|
data/lib/swineherd/script.rb
CHANGED
@@ -56,17 +56,13 @@ module Swineherd
|
|
56
56
|
# Default is to run with hadoop
|
57
57
|
#
|
58
58
|
def run mode=:hadoop
|
59
|
-
case mode
|
60
|
-
when :local then
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
sh cmd do |res, ok|
|
67
|
-
Log.info("Exit status was #{ok}")
|
68
|
-
raise "Hadoop mode script failed with exit status #{ok}" if ok != 0
|
69
|
-
end
|
59
|
+
command = case mode
|
60
|
+
when :local then local_cmd
|
61
|
+
when :hadoop then cmd
|
62
|
+
end
|
63
|
+
|
64
|
+
sh command do |ok, status|
|
65
|
+
ok or raise "#{mode.to_s.capitalize} mode script failed with exit status #{status}"
|
70
66
|
end
|
71
67
|
end
|
72
68
|
|
data/swineherd.gemspec
CHANGED
@@ -5,13 +5,13 @@
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
7
|
s.name = %q{swineherd}
|
8
|
-
s.version = "0.0.
|
8
|
+
s.version = "0.0.6"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
|
-
s.authors = ["Jacob Perkins"]
|
12
|
-
s.date = %q{2011-
|
11
|
+
s.authors = ["Jacob Perkins","Infochimps"]
|
12
|
+
s.date = %q{2011-10-05}
|
13
13
|
s.description = %q{Swineherd is for running scripts and workflows on filesystems.}
|
14
|
-
s.email =
|
14
|
+
s.email = ["jacob.a.perkins@gmail.com","coders@infochimps.com"]
|
15
15
|
s.executables = ["hdp-tree", "hadoop-stream"]
|
16
16
|
s.extra_rdoc_files = [
|
17
17
|
"LICENSE",
|
@@ -60,7 +60,6 @@ Gem::Specification.new do |s|
|
|
60
60
|
s.summary = %q{Flexible data workflow glue.}
|
61
61
|
|
62
62
|
if s.respond_to? :specification_version then
|
63
|
-
current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
|
64
63
|
s.specification_version = 3
|
65
64
|
|
66
65
|
if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
|
data/tests/testcfg.yaml
CHANGED
@@ -3,5 +3,5 @@ filesystem_to_test: file
|
|
3
3
|
s3_test_bucket: infochimps-test
|
4
4
|
|
5
5
|
# :) you'll probably want to change these
|
6
|
-
aws_access_key_id:
|
7
|
-
aws_secret_access_key:
|
6
|
+
aws_access_key_id: 02S6Y1EFA7ZZ7KCZH3G2
|
7
|
+
aws_secret_access_key: mY13mL7/iXA+gjzNJ8O95LukPQgsWjyXqVizX6zB
|
metadata
CHANGED
@@ -1,21 +1,17 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: swineherd
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
|
5
|
-
|
6
|
-
segments:
|
7
|
-
- 0
|
8
|
-
- 0
|
9
|
-
- 4
|
10
|
-
version: 0.0.4
|
4
|
+
prerelease:
|
5
|
+
version: 0.0.6
|
11
6
|
platform: ruby
|
12
7
|
authors:
|
13
8
|
- Jacob Perkins
|
9
|
+
- Infochimps
|
14
10
|
autorequire:
|
15
11
|
bindir: bin
|
16
12
|
cert_chain: []
|
17
13
|
|
18
|
-
date: 2011-
|
14
|
+
date: 2011-10-05 00:00:00 -05:00
|
19
15
|
default_executable:
|
20
16
|
dependencies:
|
21
17
|
- !ruby/object:Gem::Dependency
|
@@ -26,11 +22,6 @@ dependencies:
|
|
26
22
|
requirements:
|
27
23
|
- - ~>
|
28
24
|
- !ruby/object:Gem::Version
|
29
|
-
hash: 7
|
30
|
-
segments:
|
31
|
-
- 0
|
32
|
-
- 6
|
33
|
-
- 0
|
34
25
|
version: 0.6.0
|
35
26
|
type: :development
|
36
27
|
version_requirements: *id001
|
@@ -42,11 +33,6 @@ dependencies:
|
|
42
33
|
requirements:
|
43
34
|
- - ~>
|
44
35
|
- !ruby/object:Gem::Version
|
45
|
-
hash: 7
|
46
|
-
segments:
|
47
|
-
- 1
|
48
|
-
- 5
|
49
|
-
- 2
|
50
36
|
version: 1.5.2
|
51
37
|
type: :development
|
52
38
|
version_requirements: *id002
|
@@ -58,9 +44,6 @@ dependencies:
|
|
58
44
|
requirements:
|
59
45
|
- - ">="
|
60
46
|
- !ruby/object:Gem::Version
|
61
|
-
hash: 3
|
62
|
-
segments:
|
63
|
-
- 0
|
64
47
|
version: "0"
|
65
48
|
type: :development
|
66
49
|
version_requirements: *id003
|
@@ -72,9 +55,6 @@ dependencies:
|
|
72
55
|
requirements:
|
73
56
|
- - ">="
|
74
57
|
- !ruby/object:Gem::Version
|
75
|
-
hash: 3
|
76
|
-
segments:
|
77
|
-
- 0
|
78
58
|
version: "0"
|
79
59
|
type: :runtime
|
80
60
|
version_requirements: *id004
|
@@ -86,9 +66,6 @@ dependencies:
|
|
86
66
|
requirements:
|
87
67
|
- - ">="
|
88
68
|
- !ruby/object:Gem::Version
|
89
|
-
hash: 3
|
90
|
-
segments:
|
91
|
-
- 0
|
92
69
|
version: "0"
|
93
70
|
type: :runtime
|
94
71
|
version_requirements: *id005
|
@@ -100,9 +77,6 @@ dependencies:
|
|
100
77
|
requirements:
|
101
78
|
- - ">="
|
102
79
|
- !ruby/object:Gem::Version
|
103
|
-
hash: 3
|
104
|
-
segments:
|
105
|
-
- 0
|
106
80
|
version: "0"
|
107
81
|
type: :runtime
|
108
82
|
version_requirements: *id006
|
@@ -114,14 +88,13 @@ dependencies:
|
|
114
88
|
requirements:
|
115
89
|
- - ">="
|
116
90
|
- !ruby/object:Gem::Version
|
117
|
-
hash: 3
|
118
|
-
segments:
|
119
|
-
- 0
|
120
91
|
version: "0"
|
121
92
|
type: :runtime
|
122
93
|
version_requirements: *id007
|
123
94
|
description: Swineherd is for running scripts and workflows on filesystems.
|
124
|
-
email:
|
95
|
+
email:
|
96
|
+
- jacob.a.perkins@gmail.com
|
97
|
+
- coders@infochimps.com
|
125
98
|
executables:
|
126
99
|
- hdp-tree
|
127
100
|
- hadoop-stream
|
@@ -179,23 +152,17 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
179
152
|
requirements:
|
180
153
|
- - ">="
|
181
154
|
- !ruby/object:Gem::Version
|
182
|
-
hash: 3
|
183
|
-
segments:
|
184
|
-
- 0
|
185
155
|
version: "0"
|
186
156
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
187
157
|
none: false
|
188
158
|
requirements:
|
189
159
|
- - ">="
|
190
160
|
- !ruby/object:Gem::Version
|
191
|
-
hash: 3
|
192
|
-
segments:
|
193
|
-
- 0
|
194
161
|
version: "0"
|
195
162
|
requirements: []
|
196
163
|
|
197
164
|
rubyforge_project:
|
198
|
-
rubygems_version: 1.
|
165
|
+
rubygems_version: 1.6.2
|
199
166
|
signing_key:
|
200
167
|
specification_version: 3
|
201
168
|
summary: Flexible data workflow glue.
|