shards 0.1.0 → 1.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ca225377c101802de92a670d8d81e2d5b6d0de4a
-  data.tar.gz: 642c766b71407c5ba5e45d55dd0a7a0748340e00
+  metadata.gz: 523c7e9cab3fd380bcea554992a5c19251c84479
+  data.tar.gz: 9e928d55218cde3343ce950d6d6f2cc256cca8de
 SHA512:
-  metadata.gz: 0faf4808695321e851bff64b5cc7ba6a0d1bec68efa9981bfec4d96f73ddcb457f8e2099ab140c95dfb0971f751f1b320c4fac433dda668cb1616f247df4df43
-  data.tar.gz: 3c4fec087aeb2572ea5a70c265d711c34c5311682f091da4b9976ab18361ea32d064ee58170716fad4f451d8a35264e745703d2b9e15b82ff66ffedd3b5b3e01
+  metadata.gz: f38a73a336d92cd2a610a8a0f633cf06c03b95b5ad5484a37f86b89e0481ed3fb1ede5defda6db43d4cd80705506e3649c54b8f4a0a0b0e60073bdca2dbf6d76
+  data.tar.gz: afc4fb3664b00b44326f1eb64dde9a14579f57e815b2fff2633912c4153fd09bbc34b040b33d8f0758b5a7ec7c88f5705aa3ec2d4d7470170bdbce42a28598ff

data/.gitignore

@@ -11,3 +11,5 @@
 .rspec_status
 .byebug*
 *.swp
+settings.env
+/spec/support/write_yaml/

data/.ruby-version

@@ -0,0 +1 @@
+2.3.3

data/Gemfile.lock

@@ -0,0 +1,98 @@
+PATH
+  remote: .
+  specs:
+    shards (1.0.2)
+      aws-sdk-route53 (~> 1.9)
+      dotenv (~> 2.4)
+      git (~> 1.4)
+      net-scp (~> 1.2)
+      net-ssh (~> 5.0)
+      terminal-table (~> 1.8)
+      thor (~> 0.20)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    aruba (0.14.6)
+      childprocess (>= 0.6.3, < 0.10.0)
+      contracts (~> 0.9)
+      cucumber (>= 1.3.19)
+      ffi (~> 1.9.10)
+      rspec-expectations (>= 2.99)
+      thor (~> 0.19)
+    aws-eventstream (1.0.1)
+    aws-partitions (1.96.0)
+    aws-sdk-core (3.22.1)
+      aws-eventstream (~> 1.0)
+      aws-partitions (~> 1.0)
+      aws-sigv4 (~> 1.0)
+      jmespath (~> 1.0)
+    aws-sdk-route53 (1.10.1)
+      aws-sdk-core (~> 3)
+      aws-sigv4 (~> 1.0)
+    aws-sigv4 (1.0.3)
+    backports (3.11.3)
+    builder (3.2.3)
+    byebug (10.0.2)
+    childprocess (0.9.0)
+      ffi (~> 1.0, >= 1.0.11)
+    contracts (0.16.0)
+    cucumber (3.1.2)
+      builder (>= 2.1.2)
+      cucumber-core (~> 3.2.0)
+      cucumber-expressions (~> 6.0.1)
+      cucumber-wire (~> 0.0.1)
+      diff-lcs (~> 1.3)
+      gherkin (~> 5.1.0)
+      multi_json (>= 1.7.5, < 2.0)
+      multi_test (>= 0.1.2)
+    cucumber-core (3.2.0)
+      backports (>= 3.8.0)
+      cucumber-tag_expressions (~> 1.1.0)
+      gherkin (>= 5.0.0)
+    cucumber-expressions (6.0.1)
+    cucumber-tag_expressions (1.1.1)
+    cucumber-wire (0.0.1)
+    diff-lcs (1.3)
+    dotenv (2.5.0)
+    ffi (1.9.25)
+    gherkin (5.1.0)
+    git (1.4.0)
+    jmespath (1.4.0)
+    multi_json (1.13.1)
+    multi_test (0.1.2)
+    net-scp (1.2.1)
+      net-ssh (>= 2.6.5)
+    net-ssh (5.0.2)
+    rake (10.4.2)
+    rspec (3.6.0)
+      rspec-core (~> 3.6.0)
+      rspec-expectations (~> 3.6.0)
+      rspec-mocks (~> 3.6.0)
+    rspec-core (3.6.0)
+      rspec-support (~> 3.6.0)
+    rspec-expectations (3.6.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.6.0)
+    rspec-mocks (3.6.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.6.0)
+    rspec-support (3.6.0)
+    terminal-table (1.8.0)
+      unicode-display_width (~> 1.1, >= 1.1.1)
+    thor (0.20.0)
+    unicode-display_width (1.4.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  aruba (~> 0.14)
+  bundler (~> 1.16)
+  byebug (~> 10.0)
+  rake (~> 10.0)
+  rspec (~> 3.0)
+  shards!
+
+BUNDLED WITH
+   1.16.2

data/README.md

@@ -2,7 +2,9 @@
 
 Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/shards`. To experiment with that code, run `bin/console` for an interactive prompt.
 
-TODO: Delete this and the text above, and describe your gem
+This gem is to manage the shards and sites working with octopus in rails and sycronize between Puppet repository reading shards and sites. Then it has some AWS libraries to update route53 DNS system and create databases in RDS servers.
+
+Basically the gem has mothods to format by default some fields to save code lines in settings, and automation. The target is have a library and a terminal client application to create and delete shards and sites, and create and delete all resources associated with them.
 
 ## Installation
 
@@ -22,7 +24,36 @@ Or install it yourself as:
 
 ## Usage
 
-TODO: Write usage instructions here
+Add a settings.env file in your root application directory with the variables:
+
+```bash
+
+ENGINEERING_ROOT_PATH=/your/root/path/to/engineering/repo/project
+CONFIG_FILE=relative/path/to/config.yml
+PROXY_SUBDOMAIN=proxy.example.com
+DOMAIN=example.com
+CONFIG_FILES_ROOT_PATH=puppet/environments/common/bbug_config/files
+SHARDS_FILE_NAME=shards.yml
+SITES_FILE_NAME=sites.yml
+SHARD_LIST_KEY_ROOT_PATH=octopus production migration_group_shards
+CONFIG_DIR_SUFFIX=config_dir_suffix
+DNS_ZONE_CODE=DNS Zone Code
+DB_USER=db_user_name
+DB_PASSWORD=HerePasswordDB
+PROXY_USER=proxy_username
+
+# optional:
+TEST=true #only for test.env delete this var in settings.env
+CONFIG_REPLACE={ 'prod' => 'web' }
+LIVE_BRANCH=shards_test # set this var to use in test.env or to change the default value 'v5'
+WORKFLOW_YAML_FILE=path/to/your/workflow.yaml # write the path to your workflow yaml. You have to implement the existing methods.
+# You could overwrite methods using this metaprograming way ading your on your_file.yaml
+# Then add this var:
+# FORMAT_METHODS_FILE=your/path/to/your_file.yaml
+# Finally to overwrite the default behavior write your code in your_file.yaml. The yaml will be merged so you need define just the methods thay you need overwrite.
+# The default definitions are in spec/support/format_methods.yaml
+
+```
 
 ## Development
 
@@ -40,4 +71,4 @@ The gem is available as open source under the terms of the [MIT License](https:/
 
 ## Code of Conduct
 
-Everyone interacting in the Shards project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/[USERNAME]/shards/blob/master/CODE_OF_CONDUCT.md).
+Everyone interacting in the Shards project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/BookingBug/shards/blob/master/CODE_OF_CONDUCT.md).

data/exe/shards

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require "shards"
+
+Shards::Client.start ARGV

data/lib/settings/format_methods.yaml

@@ -0,0 +1,151 @@
+---
+Shards::Dns:
+  name: (domain + stage.get('domain_suffix')).gsub('_','-')
+  zone: ENV['DNS_ZONE_CODE']
+  target: stage.route53
+  r53upsert: >
+    {
+      change_batch: {
+        changes: [
+          {
+            action: "UPSERT",
+            resource_record_set: {
+              name: host,
+              resource_records: [
+                {
+                  value: target
+                },
+              ],
+            ttl: 300,
+            type: "CNAME",
+            },
+          },
+        ],
+      comment: "Adding #{name} to #{ENV['DOMAIN']}"
+      },
+      hosted_zone_id: "#{zone}"
+    }
+
+  host: name + '.' + ENV['DOMAIN']
+  dryrun_message: |
+    add "##### DNS Dryrun #####"
+    add "CNAME:  #{host}"
+    add "Target: #{target}"
+    add "ZoneID: #{zone}"
+    add " "
+  creating_route53_message: |
+    add "##### Setting Up DNS"
+    add "Creating Route53 entry for #{host} as a CNAME to #{target}"
+    add " "
+  aws_route53_record_message: |
+    add "Route53 Response"
+    add aws_route53_resp.to_h
+
+Shards::Db:
+  name: |
+    [stage.downcase_location, dns.domain, stage.name ].join('_').gsub('-','_')
+  server: stage.get 'db_server'
+  net: stage.get 'blank_db'
+  net_server: stage.get 'blank_db_server'
+  user: ENV['DB_USER']
+  password: ENV['DB_PASSWORD']
+  proxy: stage.get 'proxy'
+  proxy_user: ENV['PROXY_USER']
+  file: |
+    '/tmp/createdb' + dns.name + '.sh'
+  init_script: add "#!/bin/bash"
+  creating_database: |
+    add "echo \"# Creating database #{name}\""
+    add "#{conn} -e \"CREATE DATABASE #{name}\""
+  dumping_database: |
+    add "echo \"# Dumping database #{net}\""
+    add "#{dump} #{net} > #{sql_dump_file}"
+  inserting_data: |
+    add "echo \"# Inserting data from #{net} into #{name}\""
+    add "#{conn} #{name} < #{sql_dump_file}"
+  conn: |
+    "mysql #{conn_usr_pass} -h#{server}"
+  dump: |
+    "mysqldump #{conn_usr_pass} -h#{net_server}"
+  conn_usr_pass: '"-u#{user} -p#{password}"'
+  setting_up_database_message: |
+    add ""
+    add "##### Setting Up Database"
+    add "Generating DB creation script"
+  dryrun_message: |
+    add ""
+    add "##### Database Dryrun #####"
+    add "Source: #{net}"
+    add "Target: #{name}"
+  sql_dump_file: |
+    "/tmp/#{net}.sql.dump"
+  chmod_file_command: |
+    "chmod +x #{file}"
+  chmod_file_message: |
+    "Modifying Permissions."
+  db_creation_command: file
+  db_creation_message: |
+    "Running the DB Creation Script"
+  adapter: |
+    "mysql2"
+  reconnect: |
+    "true"
+  connection: |
+    { 'host' => server , 'adapter' => adapter, 'reconnect' => reconnect , 'username' => user , 'password' => password , 'database' => name }
+  show_command: |
+    conn + ' -N -B -e "show databases"'
+
+
+Shards::Site:
+  file: file_path ENV['SITES_FILE_NAME']
+  name: |
+    client + ' - ' + stage.name.capitalize
+  desc: name
+  data: |
+    {
+      'name' => name,
+      'description' => desc,
+      'subdomain' => stage.shard.dns.name,
+      'domain' => domain,
+      'shard' => stage.shard.name
+    }
+  domain: |
+    stage.shard.dns.host
+  add_site_message: |
+    add ""
+    add "##### Sites Dryrun #####"
+    data.to_s.split(separator).each { |x| add x }
+  separator: "', '"
+  add_data: |
+      yaml << data
+      yaml.sort_by! { |x| x["name"] }
+
+
+Shards::Shard:
+  file: file_path ENV['SHARDS_FILE_NAME']
+  list: eval "yaml#{shard_list_key.to_s}"
+  base: |
+    { basename => basedata }
+  basename: list.keys.first
+  basedata: list[basename]
+  name: |
+    [stage.downcase_location , domain , stage.name].join('_')
+
+Shards::Stage:
+  route53: name + '01.' + downcase_location + '.' + ENV['DOMAIN']
+  config_dir: |
+    [downcase_location, downcase_location + '_' + config_name, ENV['CONFIG_DIR_SUFFIX'] ].join('/')
+  blank_db: |
+    [downcase_location, 'net', name].join('_')
+  blank_db_server: get 'db_server'
+  proxy: downcase_location + '.' + ENV['PROXY_SUBDOMAIN']
+  domain_suffix: |
+    '-' + name
+  downcase_location: location_name.downcase
+  add_domain_message: |
+    add " "
+    add "#### Shards DryRun ####"
+    add "Shard: #{shard.name}"
+    add "DB: #{shard.db.name}"
+
+

data/lib/settings/workflow.yaml

@@ -0,0 +1,171 @@
+---
+terminal:
+
+  ask_client:
+    question: Type the client name, please.
+    wrong_message: The client name %s.
+    correct: ask_location
+    wrong: ask_for_repeat
+    min_size: 2
+
+  ask_location:
+    question: Type the location mumber, please.
+    wrong_message: The location %s is not in the list.
+    correct: ask_stage
+    wrong: ask_for_repeat
+
+  ask_stage:
+    question: Type stage number, please.
+    wrong_message: The stage %s is not in the list.
+    correct: ask_domain
+    wrong: ask_for_repeat
+
+  ask_domain:
+    question: Type the domain name, please.
+    wrong_message: The domain name %s.
+    correct: check_shard_presence_in_shards
+    wrong: ask_for_repeat
+    min_size: 3
+    regex: ^[a-zA-Z0-9_]*$
+
+  check_shard_presence_in_shards:
+    wrong_message: The shard %s exists in shards.
+    correct: check_database_presence_in_server
+    wrong: finish
+
+  check_database_presence_in_server:
+    wrong_message: The database %s exists in server %s.
+    correct: check_domain_dns_presence
+    wrong: finish
+
+  check_domain_dns_presence:
+    wrong_message: There is already a record for the host %s.
+    correct: check_domain_presence_in_sites
+    wrong: finish
+
+  check_domain_presence_in_sites:
+    wrong_message: The domain %s is already present in sites.
+    correct: write_yaml_files
+    wrong: finish
+
+  write_yaml_files:
+    question: Type Y to confirm the changes
+    wrong_message: Undo changes
+    correct: create_database
+    wrong: clean_repo_and_exit
+
+  create_database:
+    message: The database %s has been successfully created in %s.
+    wrong_message: Error. Couldn't create the database %s in %s.
+    correct: commit_changes
+    wrong: clean_repo_and_exit
+
+  commit_changes:
+    message: The changes has been committed and pushed to the remote repository.
+    wrong_message: We had an issue to commit and push the changes.
+    correct: dns_upsert
+    wrong: finish
+
+  dns_upsert:
+    message: The dns record with the host %s has been succesfully created.
+    wrong_message: Error. Could not register the host %s as a dns record.
+    correct: ask_for_exit_or_restart
+    wrong: finish
+
+  ask_for_exit_and_restart:
+    question: Type 'r' to restart the process, another key to finish.
+    wrong_message: Process Interrupted.
+    correct: ask_location
+    wrong: finish
+
+  ask_for_repeat:
+    question: Wrong answer, type 'r' to repeat the step, another key to finish.
+    correct: repeat_previous_method
+    wrong_message: Process Interrupted. Cleanning the repo and exit.
+    wrong: finish
+
+  clean_repo_and_exit:
+    correct: finish
+
+  finish:
+    correct: finish
+
+  repeat_previous_method:
+    correct: repeat_previous_method
+    wrong: finish
+
+
+fastterminal:
+
+  create:
+    correct: check_shard_presence_in_shards
+    wrong: clean_repo_and_exit
+    client_min_size: 2
+    domain_min_size: 3
+    domain_regex: ^[a-zA-Z0-9_]*$
+
+  check_shard_presence_in_shards:
+    wrong_message: The shard %s exists in shards.
+    correct: check_database_presence_in_server
+    wrong: finish
+
+  check_database_presence_in_server:
+    wrong_message: The database %s exists in server %s.
+    correct: check_domain_dns_presence
+    wrong: finish
+
+  check_domain_dns_presence:
+    wrong_message: There is already a record for the host %s.
+    correct: check_domain_presence_in_sites
+    wrong: finish
+
+  check_domain_presence_in_sites:
+    wrong_message: The domain %s is already present in sites.
+    correct: write_yaml_files
+    wrong: finish
+
+  write_yaml_files:
+    question: Type Y to confirm the changes
+    wrong_message: Undo changes
+    correct: create_database
+    wrong: clean_repo_and_exit
+
+  create_database:
+    message: The database %s has been successfully created in %s.
+    wrong_message: Error. Couldn't create the database %s in %s.
+    correct: commit_changes
+    wrong: clean_repo_and_exit
+
+  commit_changes:
+    message: The changes has been committed and pushed to the remote repository.
+    wrong_message: We had an issue to commit and push the changes.
+    correct: dns_upsert
+    wrong: finish
+
+  dns_upsert:
+    message: The dns record with the host %s has been succesfully created.
+    wrong_message: Error. Could not register the host %s as a dns record.
+    correct: ask_for_exit_or_restart
+    wrong: finish
+
+  ask_for_exit_and_restart:
+    question: Type 'r' to restart the process, another key to finish.
+    wrong_message: Process Interrupted.
+    correct: ask_location
+    wrong: finish
+
+  ask_for_repeat:
+    question: Wrong answer, type 'r' to repeat the step, another key to finish.
+    correct: repeat_previous_method
+    wrong_message: Process Interrupted. Cleanning the repo and exit.
+    wrong: finish
+
+  clean_repo_and_exit:
+    correct: finish
+
+  finish:
+    correct: finish
+
+  repeat_previous_method:
+    correct: repeat_previous_method
+    wrong: finish