rise-mirror_manager 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 5a44e98c5346580d88a87c91152a46d913fa7e4dc7e27bdc6c603037d94442f9
+  data.tar.gz: 2242749b201b99dcb73e49704d2cefc61b9dd7caa9deb9036402bb90010c2e75
+SHA512:
+  metadata.gz: 28267919621c984337ae5072e08242811621c57d1d09eb5e180fbc950cb7bb49c6db5986cc746b589c6d65d4f9d6eac57bc66e3b21d698a2c02b8033106d2ae3
+  data.tar.gz: 6b95a2d46a3eb9715d5bc29c198f95d983b9a00e34ad1f8ef8f7f08a5c91265485e1cbe3891eb5d41383e543405ac21ebfc204baf384932feddc79ea760a635c

data/.gitignore

@@ -0,0 +1,12 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+/logs
+
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,2 @@
+Metrics/LineLength:
+  Max: 120

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.4.2
+before_install: gem install bundler -v 1.16.0

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,74 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, gender identity and expression, level of experience,
+nationality, personal appearance, race, religion, or sexual identity and
+orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at script-deployer@me.com. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at [http://contributor-covenant.org/version/1/4][version]
+
+[homepage]: http://contributor-covenant.org
+[version]: http://contributor-covenant.org/version/1/4/

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in rise-mirror_manager.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,79 @@
+PATH
+  remote: .
+  specs:
+    rise-mirror_manager (0.1.0)
+      concurrent-ruby
+      inifile
+      marz-rsync (~> 0.2.0)
+      pony
+      slack-incoming-webhooks
+      sqlite3
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    backports (3.11.4)
+    builder (3.2.3)
+    coderay (1.1.2)
+    concurrent-ruby (1.1.4)
+    cucumber (3.1.2)
+      builder (>= 2.1.2)
+      cucumber-core (~> 3.2.0)
+      cucumber-expressions (~> 6.0.1)
+      cucumber-wire (~> 0.0.1)
+      diff-lcs (~> 1.3)
+      gherkin (~> 5.1.0)
+      multi_json (>= 1.7.5, < 2.0)
+      multi_test (>= 0.1.2)
+    cucumber-core (3.2.1)
+      backports (>= 3.8.0)
+      cucumber-tag_expressions (~> 1.1.0)
+      gherkin (~> 5.0)
+    cucumber-expressions (6.0.1)
+    cucumber-tag_expressions (1.1.1)
+    cucumber-wire (0.0.1)
+    diff-lcs (1.3)
+    gherkin (5.1.0)
+    inifile (3.0.0)
+    mail (2.7.1)
+      mini_mime (>= 0.1.1)
+    marz-rsync (0.2.0)
+    method_source (0.9.2)
+    mini_mime (1.0.1)
+    multi_json (1.13.1)
+    multi_test (0.1.2)
+    pony (1.12)
+      mail (>= 2.0)
+    pry (0.12.2)
+      coderay (~> 1.1.0)
+      method_source (~> 0.9.0)
+    rake (10.5.0)
+    rspec (3.8.0)
+      rspec-core (~> 3.8.0)
+      rspec-expectations (~> 3.8.0)
+      rspec-mocks (~> 3.8.0)
+    rspec-core (3.8.0)
+      rspec-support (~> 3.8.0)
+    rspec-expectations (3.8.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-mocks (3.8.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-support (3.8.0)
+    slack-incoming-webhooks (0.2.0)
+    sqlite3 (1.3.13)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.16)
+  cucumber (~> 3.0)
+  pry
+  rake (~> 10.0)
+  rise-mirror_manager!
+  rspec (~> 3.0)
+
+BUNDLED WITH
+   1.16.2

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2018 marzdgzmn
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,27 @@
+# Rise::MirrorManager
+
+Manager for Mirror Sync jobs.
+
+## Usage
+
+rise-mirror_manager/bin/mirror_manager #{interval}
+
+## Development
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Requirements
+- ruby >=
+- sqlite3 >= 3.24
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/rise-mirror_manager. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+
+## Code of Conduct
+
+Everyone interacting in the Rise::MirrorManager project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/[USERNAME]/rise-mirror_manager/blob/master/CODE_OF_CONDUCT.md).

data/Rakefile

@@ -0,0 +1,6 @@
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require File.join(__dir__, '../config/initializers/production')
+# require File.join(__dir__, '../config/initializers/test')
+require "rise/mirror_manager"
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/mirror_manager

@@ -0,0 +1,20 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require File.join(__dir__, '../config/initializers/production')
+require File.join(__dir__, '../lib/rise/mirror_manager/')
+
+if ARGV.length < 1
+  raise'Please provide at least one interval'
+  exit(1)
+end
+
+# TODO: Check if arguments are Integers
+
+# begin
+  Rise::MirrorManager.run(ARGV)
+# rescue ArgumentError
+#   Rise::MirrorManager::LOG.error "Invalid Argument: #{ARGV}"
+# end
+
+

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/config/initializers/production.rb

@@ -0,0 +1,34 @@
+require File.join(__dir__, '../../lib/rise/mirror_manager/configuration')
+require 'sqlite3'
+
+# require_relative File.join(__dir__, '../../config/initializers/production')
+# puts 'initializing'
+
+Rise::MirrorManager.configure do |config|
+  config.log_dir = '/var/log/mirror_manager/'
+  config.admin_email = 'riseops@rise.ph'
+  config.mirror_data_file = File.join(__dir__, '../../data/mirror_data.ini')
+  config.database = File.join(__dir__, '../../db/mirror_prod.db')
+  config.slack_webhook = ENV['SLACK_WEBHOOK']
+end
+
+mirror_db = Rise::MirrorManager.configuration.database
+unless File.exist?(mirror_db)
+  SQLite3::Database.open(mirror_db, results_as_hash: true) do |db|
+    db.execute %{
+      CREATE TABLE IF NOT EXISTS mirrors(
+      mirror_id INTEGER NOT NULL PRIMARY KEY,
+      name TEXT NOT NULL UNIQUE
+  );}
+    db.execute %{
+      CREATE TABLE IF NOT EXISTS sync_info(
+			sync_info_id INTEGER NOT NULL PRIMARY KEY,
+			mirror_id INTEGER UNIQUE NOT NULL,
+			sync_status TEXT,
+			total_bandwidth TEXT,
+			error TEXT,
+			FOREIGN KEY (mirror_id) REFERENCES mirror (mirror_id)
+			ON DELETE CASCADE ON UPDATE NO ACTION
+  );}
+  end
+end

data/config/initializers/test.rb

@@ -0,0 +1,40 @@
+require 'rise/mirror_manager/configuration'
+require 'sqlite3'
+
+puts 'Initializing . . .'
+
+Rise::MirrorManager.configure do |config|
+  config.log_dir = File.join(__dir__, '../../logs/')
+  config.admin_email = 'riseops@rise.ph'
+  config.mirror_data_file = File.join(__dir__, '../../spec/data/mirror_data.ini')
+  config.database = File.join(__dir__, '../../db/mirror_test.db')
+end
+
+# Create /db/mirror_test.db
+mirror_db = Rise::MirrorManager.configuration.database
+File.delete(mirror_db) if File.exist?(mirror_db)
+SQLite3::Database.open(mirror_db, results_as_hash: true) do |db|
+  db.execute %{
+      CREATE TABLE IF NOT EXISTS mirrors(
+      mirror_id INTEGER NOT NULL PRIMARY KEY,
+      name TEXT NOT NULL UNIQUE
+  );}
+  db.execute %{
+      CREATE TABLE IF NOT EXISTS sync_info(
+			sync_info_id INTEGER NOT NULL PRIMARY KEY,
+			mirror_id INTEGER UNIQUE NOT NULL,
+			sync_status TEXT,
+			total_bandwidth TEXT,
+			error TEXT,
+			FOREIGN KEY (mirror_id) REFERENCES mirror (mirror_id)
+			ON DELETE CASCADE ON UPDATE NO ACTION
+  );}
+  db.execute %{
+    INSERT INTO mirrors(name) VALUES
+    ('Ubuntu'),
+    ('ArchLinux'),
+    ('CentOS'),
+    ('Tails'),
+    ('RISE')
+  }
+end

data/data/mirror_data.ini

@@ -0,0 +1,29 @@
+; This is a test file and should NOT be used in production.
+;
+; Default rsync options: -a -q -z -u -H --no-motd --stats --safe-links --numeric-ids --delay-updates --timeout=600 --contimeout=300
+; Default rsync delete options: --delete --delete-delay
+;
+; Required configuration:
+; NAME - Mirror name
+; REMOTE_SOURCE - The source host. Not required if CUSTOM_SCRIPT config is present.
+; LOCAL_DIR - Directory to save files. Not required if CUSTOM_SCRIPT config is present.
+;
+; Optional:
+; RSYNC_OPTIONS - Override default rsync options. Do not put here the exclude, include and filter rsync options.
+; DELETE - Override default rsync delete options. Do not put here the exclude, include and filter rsync options.
+; EXCLUDE - Rsync exclude options to be added to default options.
+; INCLUDE - Rsync include options to be added to default options.
+; FILTER - Rsync filter options to be added to default options.
+; SCRIPT - Run custom script.
+; PRECMD - Command to run before start syncing.
+; POSTCMD - Command to run after the sync is finish.
+; DEPTH - Descend at most levels (a non-negative integer) of directories to search files to sync. Default is 1.
+;                    mirror sync service configuration is running on single-threaded rsync.
+
+[delete-me]
+NAME='Delete Me'
+REMOTE_SOURCE="/tmp/srv/mirror/delete-me-source"
+LOCAL_DIR="/tmp/srv/mirror/delete-me-dest"
+RSYNC_OPTS='-avzuH --no-motd --safe-links --numeric-ids  --delay-updates --timeout=600'
+DEPTH=1
+INTERVAL=4

data/lib/rise/mirror_manager.rb

@@ -0,0 +1,79 @@
+require_relative 'mirror_manager/version'
+require_relative 'mirror_manager/logging'
+require_relative 'mirror_manager/sync'
+require_relative 'mirror_manager/mirror'
+require_relative 'mirror_manager/notification'
+require 'English'
+require 'yaml'
+require 'concurrent'
+
+module Rise
+  # Module for managing mirror sync jobs
+  module MirrorManager
+
+    def self.run(intervals)
+      puts ENV['SLACK_WEBHOOK']
+      Manager.execute_jobs(intervals)
+    end
+
+    class Manager
+      include Rise::MirrorManager::Notification
+
+      LOG = Logging.setup_logger
+
+      def self.execute_jobs(intervals)
+        Manager.new(intervals).run_syncs
+      end
+
+      def run_syncs()
+        thread_pool = Concurrent::FixedThreadPool.new(10, max_queue:50)
+        until mirrors.empty?
+          mirror = mirrors.shift
+          Concurrent::Promises.future(executor: thread_pool) { Sync.run(mirror) }.rescue { |e| send_error_notification(mirror_name: mirror.name, error: e)  }.result
+        end
+        thread_pool.shutdown
+        thread_pool.wait_for_termination
+      end
+
+      private
+
+      attr_reader :mirrors, :runtime
+
+      def initialize(intervals)
+        @mirrors = retrieve_mirrors(intervals)
+        @runtime = Time.now
+      end
+
+      def send_error_notification(mirror_name:, error:)
+        send_slack_message(mirror_name: mirror_name, error: error, runtime: runtime)
+      end
+
+      def retrieve_mirrors(intervals)
+        mirrors = []
+        intervals.each do |i|
+          retrieved_mirrors = Mirror.retrieve_mirrors(Integer(i))
+          mirrors.push(*retrieved_mirrors)
+        end
+        if mirrors.empty?
+          raise 'No mirrors were retrieved from #{intervals}.'
+        end
+        mirrors
+      end
+
+      at_exit do
+        if $ERROR_INFO
+          error = {
+            timestamp: Time.now,
+            message: $ERROR_INFO.message,
+            backtrace: $ERROR_INFO.backtrace,
+            gems: Gem.loaded_specs.inject({}) do |m, (n, s)|
+            m.merge(n => s.version)
+          end
+          }
+          LOG.error YAML.dump(error)
+          send_email YAML.dump(error)
+        end
+      end
+    end
+  end
+end