mortar 0.15.50 → 0.15.51
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/mortar/command/luigi.rb +9 -18
- data/lib/mortar/command/spark.rb +6 -12
- data/lib/mortar/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 319b7fb7753bf9b969875b28eabd5d9b18291b84786bd028c328a821c91bfda4
|
|
4
|
+
data.tar.gz: 85981667afc34d06737eb2f433ee09cae0630b4d9f12548853f7ca0e4d641efc
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 358cbddf9062fe62f66471a7feca97665510d7f76af797cfc1c8a36b3ca54476febcc6a6e69f0d663ee86b1d59ca79331213edd646047d797a5fa0bc24bebfff
|
|
7
|
+
data.tar.gz: b0fe29187f701b2c9100c41d6c5577abf2507096b5026d738da1a4d825bba6c900ca5bb42e1d0df40f0373f6ac45295e27cf9eeff92003e33afbea137cee9417
|
data/lib/mortar/command/luigi.rb
CHANGED
|
@@ -27,7 +27,6 @@ class Mortar::Command::Luigi < Mortar::Command::Base
|
|
|
27
27
|
#
|
|
28
28
|
# Run a luigi pipeline.
|
|
29
29
|
#
|
|
30
|
-
# -P, --project PROJECTNAME # Use a project that is not checked out in the current directory. Runs code from project's master branch in GitHub rather than snapshotting local code.
|
|
31
30
|
# -B, --branch BRANCHNAME # Used with --project to specify a non-master branch
|
|
32
31
|
#
|
|
33
32
|
# Examples:
|
|
@@ -35,6 +34,9 @@ class Mortar::Command::Luigi < Mortar::Command::Base
|
|
|
35
34
|
# Run the nightly_rollup luigiscript:
|
|
36
35
|
# $ mortar luigi luigiscripts/nightly_rollup.py
|
|
37
36
|
#
|
|
37
|
+
# Run the nightly_rollup luigiscript from master branch:
|
|
38
|
+
# $ mortar luigi -B master luigiscripts/nightly_rollup.py
|
|
39
|
+
#
|
|
38
40
|
# Run the nightly_rollup luigiscript with two parameters:
|
|
39
41
|
# $ mortar luigi luigiscripts/nightly_rollup.py --data-date 2012-02-01 --my-param myval
|
|
40
42
|
#
|
|
@@ -43,26 +45,15 @@ class Mortar::Command::Luigi < Mortar::Command::Base
|
|
|
43
45
|
unless script_name
|
|
44
46
|
error("Usage: mortar luigi SCRIPT\nMust specify SCRIPT.")
|
|
45
47
|
end
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
script_name = File.basename(script_name, ".*")
|
|
51
|
-
end
|
|
52
|
-
else
|
|
53
|
-
project_name = project.name
|
|
54
|
-
script = validate_luigiscript!(script_name)
|
|
55
|
-
script_name = script.name
|
|
56
|
-
end
|
|
48
|
+
|
|
49
|
+
project_name = project.name
|
|
50
|
+
script = validate_luigiscript!(script_name)
|
|
51
|
+
script_name = script.name
|
|
57
52
|
|
|
58
53
|
parameters = luigi_parameters()
|
|
59
54
|
|
|
60
|
-
if options[:
|
|
61
|
-
|
|
62
|
-
git_ref = options[:branch]
|
|
63
|
-
else
|
|
64
|
-
git_ref = "master"
|
|
65
|
-
end
|
|
55
|
+
if options[:branch]
|
|
56
|
+
git_ref = options[:branch]
|
|
66
57
|
else
|
|
67
58
|
git_ref = sync_code_with_cloud()
|
|
68
59
|
end
|
data/lib/mortar/command/spark.rb
CHANGED
|
@@ -31,7 +31,6 @@ class Mortar::Command::Spark < Mortar::Command::Base
|
|
|
31
31
|
# -s, --clustersize NUMNODES # Run job with NUMNODES nodes (optional; must be >= 2 if provided)
|
|
32
32
|
# -t, --clustertags A,B,C # Run job on an existing cluster with specified tags
|
|
33
33
|
# -3, --spot # Use spot instances for this cluster (Default: true)
|
|
34
|
-
# -P, --project PROJECTNAME # Use a project that is not checked out in the current directory. Runs code from project's master branch in GitHub rather than snapshotting local code.
|
|
35
34
|
# -B, --branch BRANCHNAME # Used with --project to specify a non-master branch
|
|
36
35
|
#
|
|
37
36
|
# Examples:
|
|
@@ -39,6 +38,9 @@ class Mortar::Command::Spark < Mortar::Command::Base
|
|
|
39
38
|
# Run a spark job:
|
|
40
39
|
# $ mortar spark com.datadog.some.Job
|
|
41
40
|
#
|
|
41
|
+
# Run a spark job from master branch:
|
|
42
|
+
# $ mortar spark -B master com.datadog.some.Job
|
|
43
|
+
#
|
|
42
44
|
# Run a spark job with some arguments:
|
|
43
45
|
# $ mortar spark com.datadog.some.Job --env prod s3://your-bucket/input s3://your-bucket/output 100
|
|
44
46
|
#
|
|
@@ -48,20 +50,12 @@ class Mortar::Command::Spark < Mortar::Command::Base
|
|
|
48
50
|
error("Usage: mortar spark CLASS_NAME\nMust specify CLASS_NAME.")
|
|
49
51
|
end
|
|
50
52
|
|
|
51
|
-
|
|
52
|
-
project_name = options[:project]
|
|
53
|
-
else
|
|
54
|
-
project_name = project.name
|
|
55
|
-
end
|
|
53
|
+
project_name = project.name
|
|
56
54
|
|
|
57
55
|
script_arguments = spark_script_arguments()
|
|
58
56
|
|
|
59
|
-
if options[:
|
|
60
|
-
|
|
61
|
-
git_ref = options[:branch]
|
|
62
|
-
else
|
|
63
|
-
git_ref = "master"
|
|
64
|
-
end
|
|
57
|
+
if options[:branch]
|
|
58
|
+
git_ref = options[:branch]
|
|
65
59
|
else
|
|
66
60
|
git_ref = sync_code_with_cloud()
|
|
67
61
|
end
|
data/lib/mortar/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: mortar
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.15.
|
|
4
|
+
version: 0.15.51
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Mortar Data
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2019-04-
|
|
11
|
+
date: 2019-04-30 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: rdoc
|