swarm_orca 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 15764993dfe2fc2d4786cbec667bb3d07d3d3bda42c27a85b3383b9159f1a357
+  data.tar.gz: 43a0c48310e442f3e371a826aa37094695fc79bdfc9f058c0af553e9e282d993
+SHA512:
+  metadata.gz: c90ac38d205a8189fc9b99448afdbe59cdf0e24e852f7ae67db54a8d42027bc6a8cadca513f8a6c8c67d0837c5f622c7c9f24769a222adce353c2b4a49bbce32
+  data.tar.gz: a8956bd5d8677a23371ce317da93636d0a6956f19948dd9cf8f1252b2dd4ffc67383e8e2c8a747d0f3c9d4adbd32898d150063c5c3c16c2e8fd64e65d5d5d619

data/.gitignore

@@ -0,0 +1 @@
+.DS_Store

data/.overcommit.yml

@@ -0,0 +1,33 @@
+# Use this file to configure the Overcommit hooks you wish to use. This will
+# extend the default configuration defined in:
+# https://github.com/brigade/overcommit/blob/master/config/default.yml
+#
+# At the topmost level of this YAML file is a key representing type of hook
+# being run (e.g. pre-commit, commit-msg, etc.). Within each type you can
+# customize each hook, such as whether to only run it on certain files (via
+# `include`), whether to only display output if it fails (via `quiet`), etc.
+#
+# For a complete list of hooks, see:
+# https://github.com/brigade/overcommit/tree/master/lib/overcommit/hook
+#
+# For a complete list of options that you can use to customize hooks, see:
+# https://github.com/brigade/overcommit#configuration
+#
+# Uncomment the following lines to make the configuration take effect.
+
+PreCommit:
+ RuboCop:
+   enabled: true
+   on_warn: fail # Treat all warnings as failures
+
+ TrailingWhitespace:
+   enabled: true
+   exclude:
+     - '**/db/structure.sql' # Ignore trailing whitespace in generated files.
+
+#PostCheckout:
+#  ALL: # Special hook name that customizes all hooks of this type
+#    quiet: true # Change all post-checkout hooks to only display output on failure
+#
+#  IndexTags:
+#    enabled: true # Generate a tags file with `ctags` each time HEAD changes

data/.rubocop.yml

@@ -0,0 +1 @@
+inherit_from: .rubocop_todo.yml

data/.rubocop_todo.yml

@@ -0,0 +1,17 @@
+# This configuration was generated by
+# `rubocop --auto-gen-config`
+# on 2019-04-01 17:01:07 +0200 using RuboCop version 0.66.0.
+# The point is for the user to remove these configuration records
+# one by one as the offenses are removed from the code base.
+# Note that changes in the inspected code, or installation of new
+# versions of RuboCop, may require this file to be generated again.
+
+# Offense count: 2
+Metrics/AbcSize:
+  Max: 21
+
+# Offense count: 31
+# Configuration parameters: AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, IgnoredPatterns.
+# URISchemes: http, https
+Metrics/LineLength:
+  Max: 154

data/.ruby-gemset

@@ -0,0 +1 @@
+swarm_orca

data/.ruby-version

@@ -0,0 +1 @@
+2.6.1

data/CHANGELOG.md

@@ -0,0 +1,15 @@
+> This library uses Semantic Versioning: http://semver.org.
+
+### 0.1.0 / 2019-07-26
+First Version:
+  Features:
+     * Deploy services to a swarm cluster.
+     * Support local deployment.
+     * Debug Mode.
+     * Generate encryption keys.
+     * Support Encrypted configs.
+     * Shared application configurations
+     * Database migrations.
+     * Manage swarm services.
+     * Build custome docker images.
+     * ERB templates for stack files.

data/Gemfile

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+# Specify your gem's dependencies in enc_env.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,57 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    airbrussh (1.3.0)
+      sshkit (>= 1.6.1, != 1.7.0)
+    ast (2.4.0)
+    capistrano (3.10.2)
+      airbrussh (>= 1.0.0)
+      i18n
+      rake (>= 10.0.0)
+      sshkit (>= 1.9.0)
+    childprocess (0.9.0)
+      ffi (~> 1.0, >= 1.0.11)
+    concurrent-ruby (1.0.5)
+    ffi (1.10.0)
+    i18n (1.0.0)
+      concurrent-ruby (~> 1.0)
+    iniparse (1.4.4)
+    jaro_winkler (1.5.2)
+    net-scp (1.2.1)
+      net-ssh (>= 2.6.5)
+    net-ssh (4.2.0)
+    overcommit (0.47.0)
+      childprocess (~> 0.6, >= 0.6.3)
+      iniparse (~> 1.4)
+    parallel (1.16.0)
+    parser (2.6.2.0)
+      ast (~> 2.4.0)
+    psych (3.1.0)
+    rainbow (3.0.0)
+    rake (10.5.0)
+    rubocop (0.66.0)
+      jaro_winkler (~> 1.5.1)
+      parallel (~> 1.10)
+      parser (>= 2.5, != 2.5.1.1)
+      psych (>= 3.1.0)
+      rainbow (>= 2.2.2, < 4.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 1.6)
+    ruby-progressbar (1.10.0)
+    sshkit (1.16.0)
+      net-scp (>= 1.1.2)
+      net-ssh (>= 2.8.0)
+    thor (0.20.0)
+    unicode-display_width (1.5.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.13)
+  overcommit (~> 0.47.0)
+  rake (~> 10.0)
+  rubocop (~> 0.66.0)
+
+BUNDLED WITH
+   1.17.2

data/README.md

@@ -0,0 +1,335 @@
+# Swarm Orca
+This gem includes a set of capistrano recipes used to deploy services and rails applications to a swarm cluster.
+
+## Command Line
+### Install
+
+```sh
+$~> gem install swarm_orca
+```
+
+### Commands
+
+```sh
+orca decrypt KEY CIPHER # This Command will decrypt the given cipher
+orca encrypt KEY TEXT # This Command will encrypt the given text
+orca gen_enc_key # This Command will generate new encryption key
+orca help [COMMAND] # Describe available commands or one specific command
+orca new ORCA_DIRECTORY_NAME GIT_FORK DOCKER_NETWORK  # This Command will create a new Orca project
+```
+
+## Install
+
+Add the following to your `Gemfile`.
+
+```ruby
+group :deployment do
+  gem 'capistrano'
+  gem 'swarm_orca'
+end
+```
+
+Then run
+
+    $ bundle install
+
+## Usage
+
+- Add the following to your `Capfile`.
+
+```ruby
+require "swarm_orca"
+```
+
+
+- Add the following to your `deploy.rb`.
+
+```ruby
+require "capistrano/swarm_orca/set_global_config"
+
+set :fork, "${YOUR_FORK_NAME}"
+# example
+
+set (:db_apps_stacks_mapping), {
+  core: %w(api-backend mobile-backedn),
+  frontend: %w(),
+}
+
+require "capistrano/swarm_orca/deploy"
+require "capistrano/swarm_orca/docker"
+
+```
+
+## Define your services and application
+Swarm orca define the managed services and applications in `deploy.rb`.
+
+- service_stacks: array include the services stack names. That do not need custome docker images and do not have a database.
+- service_stacks_with_build_image: services that need a custome docker image.
+- db_apps_stacks_mapping: stacks for your rails application. keys are the stack names and values are the database application names. For example, the core stack include two database applications. While the frontend have no database application.
+```
+set (:db_apps_stacks_mapping), {
+  core: %w(api-backend mobile-backedn),
+  frontend: %w(),
+}
+```
+
+- elasticsearch_apps: application names that are connected to elasticsearch.
+
+## Define your services and applications configurations
+Swarm Orca manage the stacks and application configurations in the stage files under `capistrano/config/deploy`
+
+- each stack must have the `stack_name` to define the stack name.
+- docker image , tag and other configs can be configured in the same way. These items depend on the stack file `orca/application_stack` it self and the applications needed by the application.
+
+```
+set :mysql,
+    stack_name: 'mysql',
+    mysql_docker_image: 'mysql',
+    mysql_docker_image_tag: '5.7',
+    mysql_volume: "#{fetch(:deploy_to)}/mysql"
+```
+
+- Special config keys:
+  {application}_database_url: Database URL for the rails application.
+  docker_image_prefix: docker image prefix(host and namespace)
+  {application}_docker_image: Docker image name
+  {application}_docker_image_tag: Docker image tag.
+
+## Application Seeds
+
+- Define your application seeds
+  - By default Swarm orca gem will execute `rake db:seed` to run the seeds for your defined applications.
+  - In Addition, Swarm orca gem support custom seeds. To implement a custom seed in your Orca please follow these instruction.
+    - Create a new folder in the root directory and call it "seeds".
+    - Add a seed file for each of your application. The file name should follow this schema "${application_name}.rb"
+    - Example of seeds for backend application
+
+    ```
+    cat seeds/backend.rb
+    Rails.application.load_tasks
+    Rake::Task['roles:reseed_defaults'].execute
+    ```
+
+## Application Database migrations
+
+- Define migrations, seeds and elasticsearch reindex.
+  - To be able to execute migrations, seeds and elasticsearch reindex, you need to explicitly define the roles on your stage.
+  - Example: our backend stack is using db and elasticsearch
+
+  ```
+  server "${server}", user: "deploy", roles: %w{${stack}_db ${stack}_reindex}
+  server "${server}", user: "deploy", roles: %w{backend_db backend_reindex}
+  ```
+
+## Deployment Strategy
+By default capistrano supports only git deployment strategy to support xopy strategy do the following:
+
+- Apply these changes to your `Capfile`
+   ```diff
+     -require "capistrano/scm/git"
+     +scm = ENV.fetch('SCM', 'git')
+     +require "capistrano/scm/#{scm}"
+     +install_plugin Module.const_get("Capistrano::SCM::#{scm.capitalize}")
+   ```
+- To deploy with Copy strategy add "SCM=copy" to the command line.
+  ```
+  SCM=copy bundle exec cap ${stage} deploy:${stack}
+  ```
+
+## One Time migrations
+
+- Execute One Time migrations from orca.
+  - Orca generate a rake task for executing one time migrations for each of the defined database application.
+  - These tasks are defined like the following example.
+    ```
+    deploy:otm_${application}[${migration_name}]
+    # deploy:otm_backend[update_referral_bonus_transactions_text]
+    ```
+ - The migration task should be defined in the project source code under the name space "one_time_migrations"
+
+## Data migrations
+To support data migration for individual applications, do the following.
+  - Add a new configuration item with the following pattern "${app_name}_data_migrate" to your stack configurations. The value of the item must be true to switch db:migrate to db:migrate:with_data. See the example below for add migrate with data support to a backend application.
+
+```
+set :backend, {
+    stack_name: 'backend',
+    backend_data_migrate: 'true',
+....
+  }
+```
+
+## ERB templates
+- Setup Stacks ERB templates
+  - Create an ERB template for each of your stacks. Use the extension "erb" for your stacks.
+    - Example : docker-stack-backend.yml.erb
+  - Add the following line to deploy.rb
+    - set (:docker_erb_templates) { true }
+  - Ensure the all container environment variables are double-qouted
+
+## Shared Configuration
+- To set shared configurations that are available for all stacks, use the following syntax.
+```
+  set :shared, {
+     global_variable: 'value_123'
+  }
+```
+
+- To set shared application configurations that are available in all stacks with special key.
+Need to create shared application set like:
+```
+  set :backend_shared, {
+     environment_label: "staging",
+  }
+```
+Add special key to include shared application config to application config
+```
+  set :web_backend {
+    include_shared_config: 'backend_shared',
+  }
+  set :mobile_backend {
+    include_shared_config: 'backend_shared',
+  }
+```
+
+## Local Deployment
+
+- Support Local Deployment with Swarm Orca
+ - To Be able to deploy locally with orca, you need to implement the following changes on your orca project.
+   - Create a local stage with your configs. See below template.
+
+   ```
+     server "${server}", ENV.fetch('USER', '${user}'),
+     roles: %w{
+       swarm_manager
+     }
+     set (:deploy_to) { "${deploy_to_path}" }
+   ```
+
+     - ${server}: Can be replaced by by "localhost", "127.0.0.1" or any domain that include "local" in it.
+     - ${user}: Default it will use the ENV 'USER' value, but if this ENV 'USER' is not defined, you can also defined another/own user.
+     - ${deploy_to_path}: the destination deploy to path.
+     - You mast include at least  "swarm_manager" role.
+  - Example deploying mysql and rabbitMq
+
+  ```
+    server "127.0.0.1", ENV.fetch('USER', 'shihadeh'),
+    roles: %w{
+      mysql
+      rabbitmq
+      swarm_manager
+    }
+
+    set (:deploy_to) { "/Users/shihadeh/ggs" }
+    # set the path to the docker command.
+    set (:docker_path) { "" }
+
+    set :mysql, {
+      stack_name: 'mysql',
+      mysql_docker_image: 'mysql',
+      mysql_docker_image_tag: '5.7',
+    }
+
+    set :rabbitmq, {
+      rabbitmq_docker_image: 'rabbitmq',
+      stack_name: 'rabbitmq_fr',
+      rabbitmq_docker_image_tag: '3.6-management',
+      rabbitmq_volume: '/Users/shihadeh/ggs/rmq'
+    }
+  ```
+
+## Encrypted attribute
+To support encrypted configuration do the follwing:
+
+  - Create a new encryption key using orc cli.
+  - Use orca cli to encrypt the attributes.
+  - set the encrypted attributes (you must prefix the attribute key with 'encrypted_' ie. `encrypted_cs_database_url`). for instance
+  ```
+  set :backend,
+    stack_name: 'backend',
+    backend_docker_image: 'backend',
+    backend_docker_image_tag: 'develop',
+    encrypted_backend_database_url: 'h/UNau5AFvhxDbUkBZbPw6RBJzkTPjIMmWOQ+lQ==',
+  ```
+
+  - export the encryption key one the machine where the deployemnt scripts will be executed ie (your local machine or jenkins nodes).
+
+## Swarm Orca Deployment
+- Start deployment
+  - You can use the following commands to setup and deploy locally
+
+  ```
+  # setup and deploy, it will created dbs, and run seeds
+  bundle exec cap ${stage} deploy:development_setup
+  ```
+
+  - Build custom docker images manually
+
+  ```
+  bundle exec cap ${stage} deploy:build_images
+  ```
+
+  - Deploy individual stacks
+
+  ```
+  bundle exec cap ${stage} deploy:${stack}
+  ```
+
+  - Deployment with specific fork
+
+  Swarm orca defines a defult for for deployment. The fork value can be changed by setting the ENV 'FORK'.
+  ```
+  FORK=${forkName}  bundle exec cap ${stage} deploy:${stack}
+  ```
+  Example deploying with fork 'shihadeh'
+  ```
+  FORK=shihadeh bundle exec cap ${stage} deploy:${stack}
+  ```
+
+- Deploy more than one stack
+
+  ```
+  DEPLOYED_STACKS="mysql rabbitmq" bundle exec cap ${stage} deploy:auto
+  ```
+
+- Deploy without building docker images.
+
+```
+  BUILD_IMAGE=false bundle exec cap ${stage} deploy:${stack}
+```
+
+- Deploy all
+
+  ```
+  bundle exec cap ${stage} deploy:all
+  ```
+  - Recreate DBS for an environment
+    Use deploy:recreate_all_dbs task to return an envornment(stage) databases to the initial stage
+    Example recreating DBS for an environment(stage)
+  ```
+  bundle exec cap ${stage} deploy:recreate_all_dbs
+  ```
+
+- Deploy Debug Mode
+  ```
+  ORCA_DEBUG=true bundle exec cap ${stage} ${task}
+  ```
+
+- Deployment without cleaning up old docker images.
+  - Add the `PRUNE=false` variable to your deployment command.
+  ```
+    PRUNE=false bundle exec cap local deploy:auto
+  ```
+
+## Swarm Orca special roles
+  - swarm_manager : Nodes with this command will be used to execute deployment commands. You only need one node this role per stage.
+  - swarm_node: Swarm Orca will try to cleanup old images or containers on the nodes with this role.
+  - stack: To be able to deploy service X to a given cluster, you must to include the X as a role for a manager node in that cluster.
+  - #{stack}_db: This role indeicate where the database migration will be executed.
+  - #{stack}_reindex: This role indeicate where the elasticsearch reindexing will be executed.
+
+
+
+
+
+