travis-cron_tools 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +8 -0
- data/CODE_OF_CONDUCT.md +49 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +21 -0
- data/README.md +91 -0
- data/Rakefile +10 -0
- data/bin/console +14 -0
- data/build_readme.sh +43 -0
- data/doc/fig/architecture.dot +46 -0
- data/doc/fig/architecture.png +0 -0
- data/examples/example.rb +93 -0
- data/exe/cronjob-test-job-spawner +50 -0
- data/lib/travis-cron_tools/builder.rb +15 -0
- data/lib/travis-cron_tools/contrib/find_images_in_k8s_manifest.rb +26 -0
- data/lib/travis-cron_tools/dependency/docker.rb +47 -0
- data/lib/travis-cron_tools/dependency/git.rb +24 -0
- data/lib/travis-cron_tools/travis_api.rb +92 -0
- data/lib/travis-cron_tools/util.rb +32 -0
- data/lib/travis-cron_tools/version.rb +5 -0
- data/lib/travis-cron_tools.rb +12 -0
- data/travis-cron_tools.gemspec +25 -0
- metadata +109 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 393b057561bdf06baac307372c30ae3f9001ef32
|
4
|
+
data.tar.gz: 9a3bd784b6368f93085bb6fd4488f24f24228b32
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: ea7730fae572084bbd96f5cc5063d3dae84ca75b553a026348695bea3af3f87ad334d71be327e7893f0556f460a6540d5d84fcaa1fcd458ef2ac7bee1d291463
|
7
|
+
data.tar.gz: abb0ed076518676bd670b8a5a34f2a66e17f88254ac4e775f71f1f8dda082cef0cd274baf4d375a706e97e673f63426b4be87a883049f2c0996a34b61d60730d
|
data/.gitignore
ADDED
data/CODE_OF_CONDUCT.md
ADDED
@@ -0,0 +1,49 @@
|
|
1
|
+
# Contributor Code of Conduct
|
2
|
+
|
3
|
+
As contributors and maintainers of this project, and in the interest of
|
4
|
+
fostering an open and welcoming community, we pledge to respect all people who
|
5
|
+
contribute through reporting issues, posting feature requests, updating
|
6
|
+
documentation, submitting pull requests or patches, and other activities.
|
7
|
+
|
8
|
+
We are committed to making participation in this project a harassment-free
|
9
|
+
experience for everyone, regardless of level of experience, gender, gender
|
10
|
+
identity and expression, sexual orientation, disability, personal appearance,
|
11
|
+
body size, race, ethnicity, age, religion, or nationality.
|
12
|
+
|
13
|
+
Examples of unacceptable behavior by participants include:
|
14
|
+
|
15
|
+
* The use of sexualized language or imagery
|
16
|
+
* Personal attacks
|
17
|
+
* Trolling or insulting/derogatory comments
|
18
|
+
* Public or private harassment
|
19
|
+
* Publishing other's private information, such as physical or electronic
|
20
|
+
addresses, without explicit permission
|
21
|
+
* Other unethical or unprofessional conduct
|
22
|
+
|
23
|
+
Project maintainers have the right and responsibility to remove, edit, or
|
24
|
+
reject comments, commits, code, wiki edits, issues, and other contributions
|
25
|
+
that are not aligned to this Code of Conduct, or to ban temporarily or
|
26
|
+
permanently any contributor for other behaviors that they deem inappropriate,
|
27
|
+
threatening, offensive, or harmful.
|
28
|
+
|
29
|
+
By adopting this Code of Conduct, project maintainers commit themselves to
|
30
|
+
fairly and consistently applying these principles to every aspect of managing
|
31
|
+
this project. Project maintainers who do not follow or enforce the Code of
|
32
|
+
Conduct may be permanently removed from the project team.
|
33
|
+
|
34
|
+
This code of conduct applies both within project spaces and in public spaces
|
35
|
+
when an individual is representing the project or its community.
|
36
|
+
|
37
|
+
Instances of abusive, harassing, or otherwise unacceptable behavior may be
|
38
|
+
reported by contacting a project maintainer at maarten@moretea.nl. All
|
39
|
+
complaints will be reviewed and investigated and will result in a response that
|
40
|
+
is deemed necessary and appropriate to the circumstances. Maintainers are
|
41
|
+
obligated to maintain confidentiality with regard to the reporter of an
|
42
|
+
incident.
|
43
|
+
|
44
|
+
This Code of Conduct is adapted from the [Contributor Covenant][homepage],
|
45
|
+
version 1.3.0, available at
|
46
|
+
[http://contributor-covenant.org/version/1/3/0/][version]
|
47
|
+
|
48
|
+
[homepage]: http://contributor-covenant.org
|
49
|
+
[version]: http://contributor-covenant.org/version/1/3/0/
|
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
The MIT License (MIT)
|
2
|
+
|
3
|
+
Copyright (c) 2016 Maarten Hoogendoorn
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
13
|
+
all copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,91 @@
|
|
1
|
+
# Travis CronTools
|
2
|
+
|
3
|
+
This gem provides a library to (conditionally) spawn Travis Builds, each potentially
|
4
|
+
consisting of multiple jobs.
|
5
|
+
|
6
|
+
## Why?
|
7
|
+
Having a good Continous Integration setup is crucial for developing software.
|
8
|
+
However, running a CI test may be expensive and slow. An example is if your CI sequentially triggers
|
9
|
+
the creation of a large number of virtual machines.
|
10
|
+
|
11
|
+
Note that the number of executed tests depends on both the frequence of build triggers and the number of tests.
|
12
|
+
This tool enables the creation of a CI pipeline that can reduce both by selectively
|
13
|
+
running tests, and increase speed by introducing parallel execution of tests.
|
14
|
+
|
15
|
+
## Reducing tests & Increasing speed
|
16
|
+
We can reduce the number of tests to run over time, by:
|
17
|
+
|
18
|
+
- Filtering out test-combinations that have not changed
|
19
|
+
- Not building every change, but run tests based on time intervals
|
20
|
+
|
21
|
+
### Reducing number of tests to run
|
22
|
+
If a system is designed correctly, it is clear which parts depend on others.
|
23
|
+
We can get away with running less tests if we can filter out components that have not changed since
|
24
|
+
the last sucessfull test, and only run the tests that actually have changed.
|
25
|
+
|
26
|
+
In the area of microservices, this is easier to do than in a monolith; you just check if the
|
27
|
+
version of a depdant microservice has changed, to trigger our own build.
|
28
|
+
|
29
|
+
The filter logic is highly coupled with the application and therefore we have not made any attempt
|
30
|
+
provide a generic solution. However, we provide a DSL with helper functions that can check for
|
31
|
+
changes in Docker Image Registries in Git repositories.
|
32
|
+
|
33
|
+
### Reducing the number of build triggers
|
34
|
+
The first can be solved by just building the test suite afer a certain time interval,
|
35
|
+
such as every hour, day, or week.
|
36
|
+
|
37
|
+
Travis offers Cron jobs, that can be run daily, weekly or monthly.
|
38
|
+
Therefore, we can run the expensive tests just once a day.
|
39
|
+
|
40
|
+
### Increasing speed
|
41
|
+
A Travis build can have multiple Jobs, which typically are created via the `matrix` feature
|
42
|
+
in `.travis.yml` files.
|
43
|
+
In the DSL we provide a way to spawn new Travis Builds, each of which can have multiple Jobs.
|
44
|
+
|
45
|
+
## Walktrough
|
46
|
+
1. Travis triggers a Cron build.
|
47
|
+
2. The `.travis-cron.rb` file is run; this is a plain ruby file, which used `spawn_travis_build`
|
48
|
+
as a library.
|
49
|
+
3. This script uses the `spawn_travis_build` library to fetch information from dependent artifacts
|
50
|
+
like Docker and Git, and computes for each of them when the last change was made.
|
51
|
+
Based on the last successfull cron build, the library computes the dependencies that have been
|
52
|
+
changed.
|
53
|
+
4. Based on the changed dependencies, you compute which test commands must be run. You have
|
54
|
+
to generate the contents of a new `travis.yml`.
|
55
|
+
5. The library handles the nitty gritty details of actually creating a new build that only spawns
|
56
|
+
the relevant jobs.
|
57
|
+
|
58
|
+
In the example below, there are three potential tests. After computing the changed dependencies,
|
59
|
+
it is determined that only two of the three tests have to run.
|
60
|
+
|
61
|
+
<img src="doc/fig/architecture.png">
|
62
|
+
|
63
|
+
## How to use these tools
|
64
|
+
See the [example](example/example.rb) for some possibilities.
|
65
|
+
|
66
|
+
I suggest to integrate this in your project by doing the following steps:
|
67
|
+
- Enable the Travis Cron feature, set it to daily.
|
68
|
+
- Write logic in Ruby, using the tools provided by this gem.
|
69
|
+
- Add `if [[ $TRAVIS_JOB == "cron" ]]; then ruby .travis-cron.rb; fi` to `.travis.yml`
|
70
|
+
where `.travis-cron.rb` is the file that contains your logic.
|
71
|
+
|
72
|
+
## Installation
|
73
|
+
Install the the gem by running `$ gem install spawn_travis_build`
|
74
|
+
|
75
|
+
## Development
|
76
|
+
After checking out the repo, run `bundle install` to install gem dependencies.
|
77
|
+
Then, run `rake test` to run the tests.
|
78
|
+
You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
79
|
+
|
80
|
+
To install this gem onto your local machine, run `bundle exec rake install`.
|
81
|
+
To release a new version, update the version number in `version.rb`, and then run
|
82
|
+
`bundle exec rake release`, which will create a git tag for the version, push git commits and tags,
|
83
|
+
and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
84
|
+
|
85
|
+
## Contributing
|
86
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/microservices-demo/spawn_travis_build.
|
87
|
+
This project is intended to be a safe, welcoming space for collaboration, and contributors are
|
88
|
+
expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
|
89
|
+
|
90
|
+
## License
|
91
|
+
The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
|
data/Rakefile
ADDED
data/bin/console
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require "bundler/setup"
|
4
|
+
require "spawn_travis_build"
|
5
|
+
|
6
|
+
# You can add fixtures and/or initialization code here to make experimenting
|
7
|
+
# with your gem easier. You can also use a different console, if you like.
|
8
|
+
|
9
|
+
# (If you use this, don't forget to add pry to your Gemfile!)
|
10
|
+
# require "pry"
|
11
|
+
# Pry.start
|
12
|
+
|
13
|
+
require "irb"
|
14
|
+
IRB.start
|
data/build_readme.sh
ADDED
@@ -0,0 +1,43 @@
|
|
1
|
+
#!/usr/bin/env nix-shell
|
2
|
+
#!nix-shell -i bash -p inotify-tools graphviz pandoc
|
3
|
+
|
4
|
+
build_dot() {
|
5
|
+
file=$1
|
6
|
+
echo -n "Building ${file} on "
|
7
|
+
date
|
8
|
+
dot $file -Tpdf > ${file%.dot}.pdf
|
9
|
+
dot $file -Tpng > ${file%.dot}.png
|
10
|
+
echo done
|
11
|
+
}
|
12
|
+
|
13
|
+
build_md() {
|
14
|
+
file=$1
|
15
|
+
echo -n "Building ${file} on"
|
16
|
+
date
|
17
|
+
pandoc $file -o ${file%.md}.html
|
18
|
+
echo done
|
19
|
+
}
|
20
|
+
|
21
|
+
|
22
|
+
build() {
|
23
|
+
file=$1
|
24
|
+
ext=${file##*.}
|
25
|
+
|
26
|
+
case $ext in
|
27
|
+
"md") build_md $file ;;
|
28
|
+
"dot") build_dot $file ;;
|
29
|
+
*) echo "CANNOT BUILD '$ext' extension" ;;
|
30
|
+
esac
|
31
|
+
}
|
32
|
+
|
33
|
+
files=$(echo README.md doc/fig/*.dot)
|
34
|
+
|
35
|
+
for file in $files; do
|
36
|
+
build $file
|
37
|
+
done
|
38
|
+
|
39
|
+
while true; do
|
40
|
+
inotifywait -s $files | while read -r file action; do
|
41
|
+
build $file
|
42
|
+
done
|
43
|
+
done
|
@@ -0,0 +1,46 @@
|
|
1
|
+
digraph spawn_travis_build_architect {
|
2
|
+
// edge[minlen=2];
|
3
|
+
|
4
|
+
travis_cron_trigger[label="Travis daily\ncron service",style="filled", color="#90305A", fontcolor=white];
|
5
|
+
|
6
|
+
{
|
7
|
+
rank=same;
|
8
|
+
github[shape="hexagon", label="GitHub\nrepository", style="filled", color="#6FA698", fontcolor=white];
|
9
|
+
travis_cron_job[shape="egg", label="Travis Build\n(Cron Job)", style="filled", color="#D890B0", fontcolor=white];
|
10
|
+
docker_registry[shape="hexagon", label="Docker\nregistry", style="filled", color="#6FA698", fontcolor=white];
|
11
|
+
travis_api[shape="hexagon", label="Travis API", style="filled", color="#6FA698", fontcolor=white];
|
12
|
+
docker_registry -> travis_cron_job [constraints=false, style=invis];
|
13
|
+
github -> travis_api [style=invis];
|
14
|
+
}
|
15
|
+
|
16
|
+
travis_cron_trigger -> travis_cron_job [label="1. triggers"];
|
17
|
+
|
18
|
+
travis_cron_job -> determine_changes;
|
19
|
+
docker_registry -> determine_changes[dir=back, label="3a. Pull images"];
|
20
|
+
determine_changes -> github[label="3b. Pull current master"];
|
21
|
+
travis_api -> determine_changes[dir=back, label="3c. Determine last\nsuccessful build"];
|
22
|
+
|
23
|
+
|
24
|
+
subgraph cluster_cron_job_content {
|
25
|
+
margin=20;
|
26
|
+
determine_changes[label="2. Determine changes\n(with DSL)"];
|
27
|
+
compute_tests_to_run[label="4.Filter tests to run\n(custom logic)"];
|
28
|
+
spawn_travis_build[label="5. Spawn Travis Build"];
|
29
|
+
}
|
30
|
+
|
31
|
+
|
32
|
+
determine_changes -> compute_tests_to_run[label="Changed repos &\nDocker images"];
|
33
|
+
compute_tests_to_run -> spawn_travis_build;
|
34
|
+
|
35
|
+
travis_build[shape=egg,label="Travis Build\n(only with relevant tests)", style="filled", color="#D890B0", fontcolor=white];
|
36
|
+
|
37
|
+
travis_job1[label="Test A", style="filled", fillcolor="#D890B0"];
|
38
|
+
travis_job2[label="Test B", style="filled", fillcolor="#D890B0"];
|
39
|
+
travis_job3[label="Test C", color="grey", fontcolor="grey", fontcolor="black"];
|
40
|
+
|
41
|
+
spawn_travis_build -> travis_build;
|
42
|
+
|
43
|
+
travis_build -> travis_job1;
|
44
|
+
travis_build -> travis_job2;
|
45
|
+
travis_build -> travis_job3 [color="grey",style="dotted"];
|
46
|
+
}
|
Binary file
|
data/examples/example.rb
ADDED
@@ -0,0 +1,93 @@
|
|
1
|
+
require "travis-cron_tools"
|
2
|
+
|
3
|
+
# Instantiate travis client for this repository.
|
4
|
+
travis = Travis::CronTools::TravisAPI.new("microservices-demo", "microservices-demo")
|
5
|
+
|
6
|
+
###############################################################################
|
7
|
+
# Find time to compare changes against
|
8
|
+
###############################################################################
|
9
|
+
|
10
|
+
# Find time to compare changes against
|
11
|
+
# This guarrantees that we won't miss any changes.
|
12
|
+
|
13
|
+
last_cron_job = travis.builds(event_type: "cron").first
|
14
|
+
last_cron_job_start_time = Time.parse(last_cron_job["started_at"])
|
15
|
+
|
16
|
+
last_cron_job_start_time = (Date.today - 30).to_time
|
17
|
+
|
18
|
+
puts "Checking for changes after #{last_cron_job_start_time.rfc2822}"
|
19
|
+
|
20
|
+
###############################################################################
|
21
|
+
# Find dependencies
|
22
|
+
###############################################################################
|
23
|
+
|
24
|
+
# Declare the latest nginx docker image to be a dependency
|
25
|
+
nginx_dep = Travis::CronTools::Dependency::Docker.new("nginx")
|
26
|
+
|
27
|
+
# It is also possible to specify a specific image tag
|
28
|
+
mysql= Travis::CronTools::Dependency::Docker.new("mysql", "8.0")
|
29
|
+
|
30
|
+
# In the contrib module, there are some helper methods.
|
31
|
+
# Below for example, it extracts all image names from a k8s manifest file.
|
32
|
+
k8s_docker_images = Travis::CronTools::Contrib.find_images_in_k8s_manifest("path/to/k8s.yml")
|
33
|
+
|
34
|
+
all_docker_images = k8s_docker_images + [nginx_dep, mysql]
|
35
|
+
|
36
|
+
# Domain specific struct to encode a custom dependency
|
37
|
+
Platform = Struct.new(:name, :dir)
|
38
|
+
|
39
|
+
# and compute the domain specific dependencies
|
40
|
+
platforms = Dir["deploy/*"] \
|
41
|
+
.select { |file_name| File.directory? file_name } \
|
42
|
+
.map { |file_name| Platform.new(File.basename(file_name), file_name) }
|
43
|
+
|
44
|
+
###############################################################################
|
45
|
+
# Determine which tests to run
|
46
|
+
###############################################################################
|
47
|
+
|
48
|
+
platforms_to_test = []
|
49
|
+
|
50
|
+
# Implement custom logic to determine what should be tested.
|
51
|
+
# In this case, if any of the docker images has changed, we want to test
|
52
|
+
# all different platforms we can deploy to.
|
53
|
+
if all_docker_images.any? { |image| image.created_since?(last_cron_job_start_time) }
|
54
|
+
puts "Some of the docker images changed; building all platforms"
|
55
|
+
platforms_to_test = platforms
|
56
|
+
else
|
57
|
+
# Alternatively, if none of the images has changed, we check if any deployment has changed
|
58
|
+
# in the git repository since the last successfull build.
|
59
|
+
puts "None of the docker images changed."
|
60
|
+
platforms.each do |platform|
|
61
|
+
git_dir = SpawnTravisBuild::Dependency::Git.new(platform.dir)
|
62
|
+
if git_dir.changed_since?(last_cron_job_start_time)
|
63
|
+
puts "However, deployment platform #{platform.name} changed"
|
64
|
+
platforms_to_test.push platform
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
###############################################################################
|
70
|
+
# Create Travis Build
|
71
|
+
###############################################################################
|
72
|
+
|
73
|
+
# Now we create the travis build.
|
74
|
+
# Note that we use the reset_dot_travis_yml helper; this configuration is MERGED with
|
75
|
+
# the existing configuration
|
76
|
+
|
77
|
+
cron_build_number = ENV["TRAVIS_BUILD_NUMBER"]
|
78
|
+
travis_build_request = {
|
79
|
+
message: "[deployment-daily-ci] spanwed by cron build #{cron_build_number}",
|
80
|
+
branch: ENV["TRAVIS_COMMIT"],
|
81
|
+
config: Travis::CronTools.reset_dot_travis_yml.merge({
|
82
|
+
language: "generic",
|
83
|
+
sudo: true,
|
84
|
+
services: ["docker"],
|
85
|
+
env: {
|
86
|
+
matrix: platforms_to_test.map { |platform| "PLATFORM_TO_TEST=#{platform.name}" }
|
87
|
+
},
|
88
|
+
script: ["run_testsuite --test $PLATFORM_TO_TEST"]
|
89
|
+
})
|
90
|
+
}
|
91
|
+
|
92
|
+
p travis_build_request
|
93
|
+
#travis.create_request(travis_build_request)
|
@@ -0,0 +1,50 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require_relative "../lib/all"
|
4
|
+
|
5
|
+
if ENV["TRAVIS_TOKEN"].nil?
|
6
|
+
$stderr.puts "environment variable TRAVIS_TOKEN is not defined."
|
7
|
+
exit 1
|
8
|
+
end
|
9
|
+
|
10
|
+
travis = Travis.new(ENV["TRAVIS_TOKEN"], "microservices-demo", "microservices-demo")
|
11
|
+
|
12
|
+
# Find the last cron job build.
|
13
|
+
last_cron_job = travis.builds(event_type: "cron").first
|
14
|
+
|
15
|
+
check_changes_since = if last_cron_job
|
16
|
+
s = Time.parse(last_cron_job["started_at"])
|
17
|
+
log(:info, "Last cron job ran on #{s.rfc2822}")
|
18
|
+
s
|
19
|
+
else
|
20
|
+
s = (Date.today - 30).to_time
|
21
|
+
log(:info, "No cron job found! Checking since #{s.rfc2822}")
|
22
|
+
s
|
23
|
+
end
|
24
|
+
|
25
|
+
|
26
|
+
# Find all DeployDoc's in $GIT_ROOT/doc/deployment.
|
27
|
+
documented_deployments = DeployDocTest.find_all
|
28
|
+
|
29
|
+
tests_to_run = []
|
30
|
+
|
31
|
+
if any_image_changed_since?(find_images_in_k8s_complete_demo, check_changes_since)
|
32
|
+
documented_deployments.each do |dd|
|
33
|
+
tests_to_run << dd
|
34
|
+
end
|
35
|
+
else
|
36
|
+
documented_deployments.each do |dd|
|
37
|
+
if deployment_changed?(dd, check_changes_since)
|
38
|
+
tests_to_run << dd
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
if tests_to_run.empty?
|
44
|
+
log(:info, "Nothing has changed, no tests builds will be spawned.")
|
45
|
+
else
|
46
|
+
log(:info, "Will spawn #{tests_to_run.length} test builds.")
|
47
|
+
log(:debug, "To be precise, will spawn: #{tests_to_run.inspect}.")
|
48
|
+
|
49
|
+
spawn_travis_tests(travis, tests_to_run)
|
50
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
module Travis
|
2
|
+
module CronTools
|
3
|
+
module Contrib
|
4
|
+
def self.find_images_in_k8s_manifest(*manifest_paths)
|
5
|
+
images = []
|
6
|
+
|
7
|
+
manifest_paths.each do |manifest_path|
|
8
|
+
# A manifest may contain multiple YAML 'documents'
|
9
|
+
yaml_parts = File.read(manifest_path)\
|
10
|
+
.split("---\n")\
|
11
|
+
.map { |part| YAML.load(part) }
|
12
|
+
|
13
|
+
# Extract the image name, and append ':latest', unless a tag has already been provided.
|
14
|
+
images += yaml_parts\
|
15
|
+
.select { |part| part["kind"] == "Deployment" }\
|
16
|
+
.map { |part| part["spec"]["template"]["spec"]["containers"].map { |c| c["image"] } }.flatten\
|
17
|
+
.map { |name| if name.include?(":") then name; else name + ":latest"; end }
|
18
|
+
end
|
19
|
+
|
20
|
+
images.uniq.map do |name|
|
21
|
+
Dependency::Docker.new(*name.split(":"))
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
module Travis
|
2
|
+
module CronTools
|
3
|
+
module Dependency
|
4
|
+
class Docker
|
5
|
+
class DockerError < RuntimeError
|
6
|
+
end
|
7
|
+
|
8
|
+
attr_reader :name
|
9
|
+
attr_reader :tag
|
10
|
+
|
11
|
+
|
12
|
+
def initialize(name, tag="latest")
|
13
|
+
@name = name
|
14
|
+
@tag = tag
|
15
|
+
end
|
16
|
+
|
17
|
+
def created_since?(time)
|
18
|
+
created_at > time
|
19
|
+
end
|
20
|
+
|
21
|
+
def created_at
|
22
|
+
Time.parse(metadata["Created"])
|
23
|
+
end
|
24
|
+
|
25
|
+
def metadata
|
26
|
+
@metadata ||= begin
|
27
|
+
log, status = Open3.capture2e("docker pull #{full_name}")
|
28
|
+
if !status.success?
|
29
|
+
raise DockerError.new("Could not pull #{full_name}; #{log}")
|
30
|
+
end
|
31
|
+
|
32
|
+
output, status = Open3.capture2e("docker inspect #{full_name}")
|
33
|
+
if !status.success?
|
34
|
+
raise err.with_updated_message("Could pull, but not inspect docker image #{name}:#{tag}; #{output}")
|
35
|
+
end
|
36
|
+
|
37
|
+
JSON.load(output).first
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def full_name
|
42
|
+
"#{name}:#{tag}"
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
module Travis
|
2
|
+
module CronTools
|
3
|
+
module Dependency
|
4
|
+
class Git
|
5
|
+
# It's possible to provide any number of file names
|
6
|
+
def initialize(*file_names)
|
7
|
+
@file_names = file_names.flatten
|
8
|
+
end
|
9
|
+
|
10
|
+
def changed_since?(time)
|
11
|
+
last_changed_at > time
|
12
|
+
end
|
13
|
+
|
14
|
+
def last_changed_at
|
15
|
+
@last_changed_at ||= begin
|
16
|
+
capture_or("git log -n1 --pretty=format:%cD -- #{@file_names.join(" ")}") do |err|
|
17
|
+
raise err.with_updated_message("Could not find last changes for #{file_names.inspect}")
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,92 @@
|
|
1
|
+
require "json"
|
2
|
+
require "net/http"
|
3
|
+
require "uri"
|
4
|
+
|
5
|
+
module Travis
|
6
|
+
module CronTools
|
7
|
+
class TravisAPI
|
8
|
+
attr_reader :organization, :repo
|
9
|
+
|
10
|
+
class TravisError < RuntimeError
|
11
|
+
end
|
12
|
+
|
13
|
+
def initialize(organization, repo, auth_token = :from_env)
|
14
|
+
auth_token = if auth_token == :from_env
|
15
|
+
ENV["TRAVIS_TOKEN"]
|
16
|
+
else
|
17
|
+
auth_token
|
18
|
+
end
|
19
|
+
|
20
|
+
@organization = organization
|
21
|
+
@repo = repo
|
22
|
+
@base_url = "https://api.travis-ci.org/repo/#{@organization}%2F#{@repo}"
|
23
|
+
|
24
|
+
@headers = {
|
25
|
+
"Content-Type" => "application/json",
|
26
|
+
"Accept" => "application/json",
|
27
|
+
"Travis-API-Version" => "3",
|
28
|
+
"Authorization" => "token #{auth_token}"
|
29
|
+
}
|
30
|
+
end
|
31
|
+
|
32
|
+
# Lazily fetches all builds.
|
33
|
+
def builds(filters = {})
|
34
|
+
fetch_resources_lazily("builds", filters)
|
35
|
+
end
|
36
|
+
|
37
|
+
def create_request(request_options)
|
38
|
+
uri = URI.parse("#{@base_url}/requests")
|
39
|
+
http = Net::HTTP.new(uri.host, uri.port)
|
40
|
+
http.use_ssl = true
|
41
|
+
request = Net::HTTP::Post.new(uri.request_uri, @headers)
|
42
|
+
request.body = JSON.dump(request: request_options)
|
43
|
+
response = http.request(request)
|
44
|
+
json = JSON.load response.body
|
45
|
+
|
46
|
+
if response.code != "202"
|
47
|
+
raise TravisError.new("Travis API error #{response.code}: #{json.inspect}")
|
48
|
+
end
|
49
|
+
|
50
|
+
json
|
51
|
+
end
|
52
|
+
|
53
|
+
private
|
54
|
+
|
55
|
+
def fetch_resources_lazily(resource, filters = {})
|
56
|
+
start_url = "#{@base_url}/#{resource}"
|
57
|
+
if filters.any?
|
58
|
+
start_url += "?"
|
59
|
+
end
|
60
|
+
|
61
|
+
start_url += filters.map {|filter_name, filter_value| filter_name.to_s + "=" + filter_value.to_s}.join("&")
|
62
|
+
|
63
|
+
uri = URI.parse(start_url)
|
64
|
+
|
65
|
+
Enumerator.new do |yielder|
|
66
|
+
loop do
|
67
|
+
http = Net::HTTP.new(uri.host, uri.port)
|
68
|
+
http.use_ssl = true
|
69
|
+
request = Net::HTTP::Get.new(uri.request_uri, @headers)
|
70
|
+
response = http.request(request)
|
71
|
+
|
72
|
+
if response.code != "200"
|
73
|
+
raise TravisError.new("Could not perform request; got #{response.code} code! (body: #{(request.body rescue nil).inspect})")
|
74
|
+
end
|
75
|
+
|
76
|
+
json = JSON.load response.body
|
77
|
+
|
78
|
+
json["builds"].each do |build|
|
79
|
+
yielder << build
|
80
|
+
end
|
81
|
+
|
82
|
+
next_url = json["@pagination"]["next"]
|
83
|
+
break if next_url.nil?
|
84
|
+
|
85
|
+
# the URL given by the API is relative. Past the domain in front of it.
|
86
|
+
uri = URI.parse("https://api.travis-ci.org#{next_url["@href"]}")
|
87
|
+
end
|
88
|
+
end.lazy
|
89
|
+
end
|
90
|
+
end
|
91
|
+
end
|
92
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
module Travis
|
2
|
+
module CronTools
|
3
|
+
def self.running_on_travis?
|
4
|
+
ENV.has_key?("TRAVIS_EVENT_TYPE")
|
5
|
+
end
|
6
|
+
|
7
|
+
DOT_TRAVIS_YML_WHITELIST = ["notifications"]
|
8
|
+
def self.reset_dot_travis_yml(path=".", whitelist=DOT_TRAVIS_YML_WHITELIST)
|
9
|
+
reset = {}
|
10
|
+
yaml = YAML.load(File.read(File.join(path, ".travis.yml")))
|
11
|
+
|
12
|
+
yaml.each do |key, value|
|
13
|
+
if DOT_TRAVIS_YML_WHITELIST.include?(key)
|
14
|
+
reset[key] = value
|
15
|
+
else
|
16
|
+
reset_value = case value
|
17
|
+
when Hash then {}
|
18
|
+
when Array then []
|
19
|
+
when FalseClass then nil
|
20
|
+
when TrueClass then nil
|
21
|
+
when String then nil
|
22
|
+
when Numeric then nil
|
23
|
+
else raise("Unknown reset value for #{value.class} #{value.inspect}")
|
24
|
+
end
|
25
|
+
reset[key] = reset_value
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
reset
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,12 @@
|
|
1
|
+
require "time"
|
2
|
+
require "date"
|
3
|
+
require "json"
|
4
|
+
require "yaml"
|
5
|
+
require "open3"
|
6
|
+
|
7
|
+
require "travis-cron_tools/version"
|
8
|
+
require "travis-cron_tools/util"
|
9
|
+
require "travis-cron_tools/travis_api"
|
10
|
+
require "travis-cron_tools/dependency/git"
|
11
|
+
require "travis-cron_tools/dependency/docker"
|
12
|
+
require "travis-cron_tools/contrib/find_images_in_k8s_manifest"
|
@@ -0,0 +1,25 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'travis-cron_tools/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "travis-cron_tools"
|
8
|
+
spec.version = Travis::CronTools::VERSION
|
9
|
+
spec.authors = ["Maarten Hoogendoorn"]
|
10
|
+
spec.email = ["maarten@moretea.nl"]
|
11
|
+
|
12
|
+
spec.summary = %q{Travis Cron tools}
|
13
|
+
spec.description = %q{Running expensive integration tests? With Travis Cron tools you can still have daily integration, but be selective about the tests you want to run}
|
14
|
+
spec.homepage = "https://github.com/microservices-demo/travis-cron_tools"
|
15
|
+
spec.license = "MIT"
|
16
|
+
|
17
|
+
spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
|
18
|
+
spec.bindir = "exe"
|
19
|
+
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
20
|
+
spec.require_paths = ["lib"]
|
21
|
+
|
22
|
+
spec.add_development_dependency "bundler", "~> 1.12"
|
23
|
+
spec.add_development_dependency "rake", "~> 10.0"
|
24
|
+
spec.add_development_dependency "minitest", "~> 5.0"
|
25
|
+
end
|
metadata
ADDED
@@ -0,0 +1,109 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: travis-cron_tools
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Maarten Hoogendoorn
|
8
|
+
autorequire:
|
9
|
+
bindir: exe
|
10
|
+
cert_chain: []
|
11
|
+
date: 2016-12-08 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: bundler
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '1.12'
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '1.12'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: rake
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '10.0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '10.0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: minitest
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '5.0'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '5.0'
|
55
|
+
description: Running expensive integration tests? With Travis Cron tools you can still
|
56
|
+
have daily integration, but be selective about the tests you want to run
|
57
|
+
email:
|
58
|
+
- maarten@moretea.nl
|
59
|
+
executables:
|
60
|
+
- cronjob-test-job-spawner
|
61
|
+
extensions: []
|
62
|
+
extra_rdoc_files: []
|
63
|
+
files:
|
64
|
+
- ".gitignore"
|
65
|
+
- CODE_OF_CONDUCT.md
|
66
|
+
- Gemfile
|
67
|
+
- LICENSE.txt
|
68
|
+
- README.md
|
69
|
+
- Rakefile
|
70
|
+
- bin/console
|
71
|
+
- build_readme.sh
|
72
|
+
- doc/fig/architecture.dot
|
73
|
+
- doc/fig/architecture.png
|
74
|
+
- examples/example.rb
|
75
|
+
- exe/cronjob-test-job-spawner
|
76
|
+
- lib/travis-cron_tools.rb
|
77
|
+
- lib/travis-cron_tools/builder.rb
|
78
|
+
- lib/travis-cron_tools/contrib/find_images_in_k8s_manifest.rb
|
79
|
+
- lib/travis-cron_tools/dependency/docker.rb
|
80
|
+
- lib/travis-cron_tools/dependency/git.rb
|
81
|
+
- lib/travis-cron_tools/travis_api.rb
|
82
|
+
- lib/travis-cron_tools/util.rb
|
83
|
+
- lib/travis-cron_tools/version.rb
|
84
|
+
- travis-cron_tools.gemspec
|
85
|
+
homepage: https://github.com/microservices-demo/travis-cron_tools
|
86
|
+
licenses:
|
87
|
+
- MIT
|
88
|
+
metadata: {}
|
89
|
+
post_install_message:
|
90
|
+
rdoc_options: []
|
91
|
+
require_paths:
|
92
|
+
- lib
|
93
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
94
|
+
requirements:
|
95
|
+
- - ">="
|
96
|
+
- !ruby/object:Gem::Version
|
97
|
+
version: '0'
|
98
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
99
|
+
requirements:
|
100
|
+
- - ">="
|
101
|
+
- !ruby/object:Gem::Version
|
102
|
+
version: '0'
|
103
|
+
requirements: []
|
104
|
+
rubyforge_project:
|
105
|
+
rubygems_version: 2.6.6
|
106
|
+
signing_key:
|
107
|
+
specification_version: 4
|
108
|
+
summary: Travis Cron tools
|
109
|
+
test_files: []
|