record_store 0.3.0 → 1.0.0

data/Rakefile

@@ -1 +1,18 @@
-require "bundler/gem_tasks"
+require 'rake/testtask'
+require 'bundler/gem_tasks'
+
+$LOAD_PATH.unshift(__dir__ + "/lib")
+require 'record_store'
+
+Rake::TestTask.new do |t|
+  t.libs << "test"
+  t.test_files = FileList['test/**/*_test.rb']
+end
+
+task :validate do
+  record_store = RecordStore::CLI.new
+  record_store.validate_records
+  record_store.validate_all_present
+end
+
+task default: [:test]

data/bin/record-store

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+$:.unshift File.expand_path('../../lib', __FILE__)
+require 'bundler/setup'
+require 'record_store'
+
+RecordStore::CLI.start

data/bin/setup

@@ -2,6 +2,12 @@
 set -euo pipefail
 IFS=$'\n\t'
 
-bundle install
+cd "$(dirname "$0")/.."
+ROOT_DIR=$(pwd)
 
-# Do any other automated setup that you need to do here
+bundle check || bundle install
+
+if [ ! -d $ROOT_DIR/dev ]; then
+  echo 'Created development directory dev/'
+  cp -R template dev
+fi

data/bin/test

@@ -0,0 +1,5 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle exec rake test

data/circle.yml

@@ -0,0 +1,8 @@
+machine:
+  ruby:
+    version: 2.1.1
+
+test:
+  override:
+    - bin/setup
+    - bundle exec rake test

data/lib/record_store.rb

@@ -1,122 +1,66 @@
-class RecordStore
-  VERSION = "0.3.0"
+require 'json'
+require 'yaml'
+require 'active_support'
+require 'active_support/core_ext/hash/keys'
+require 'active_support/core_ext/string'
+require 'active_model'
+require 'active_model/validations'
+require 'ipaddr'
+require 'thor'
+require 'pathname'
+
+require 'record_store/record'
+require 'record_store/record/a'
+require 'record_store/record/aaaa'
+require 'record_store/record/alias'
+require 'record_store/record/cname'
+require 'record_store/record/mx'
+require 'record_store/record/ns'
+require 'record_store/record/txt'
+require 'record_store/record/spf'
+require 'record_store/record/srv'
+require 'record_store/zone'
+require 'record_store/zone/config'
+require 'record_store/changeset'
+require 'record_store/provider'
+require 'record_store/provider/dynect'
+require 'record_store/provider/dnsimple'
+require 'record_store/cli'
+
+module RecordStore
+  MAXIMUM_REMOVALS = 20
 
   class << self
-    def database
-      raise "You must setup your database"
-    end
-
-    def dataset_name
-      @dataset_name ||= name.sub(/Store$/,'').tableize.to_sym
-    end
-
-    def dataset
-      database[dataset_name]
-    end
+    attr_writer :secrets_path
+    attr_writer :zones_path
 
-    def get(id)
-      dataset.where(id: id).first
+    def secrets_path
+      @secrets_path ||= File.expand_path(config.fetch('secrets_path'), File.dirname(config_path))
     end
-    alias_method :[], :get
 
-    def put(record)
-      new(record).put
+    def zones_path
+      @zones_path ||= Pathname.new(File.expand_path(config.fetch('zones_path'), File.dirname(config_path))).realpath.to_s
     end
-    alias_method :<<, :put
 
-    def type_column_map(database)
-      @type_column_map ||= database.schema(dataset_name).each_with_object({}) do |(column,metadata),map|
-        map[metadata[:db_type]] ||= []
-        map[metadata[:db_type]] << column
-      end
+    def config_path
+      @config_path ||= File.expand_path('config.yml', Dir.pwd)
     end
 
-    def validations
-      @validations ||= []
+    def config_path=(config_path)
+      @config = @zones_path = @secrets_path = nil
+      @config_path = config_path
     end
 
-    def required_attributes(*record)
-      record.each do |attribute|
-        validations << proc do
-          errors << "#{attribute.to_s.titleize} is required" unless @record[attribute].present?
-        end
-      end
+    def config
+      @config ||= YAML.load_file(config_path)
     end
 
-    def required_foreign_key(attribute, table)
-      validations << proc do
-        if @record[attribute].blank?
-          errors << "#{attribute} is required"
-        elsif database[table].where(id: @record[attribute]).get(:id).blank?
-          errors << "no #{table.to_s.singularize} with id=#{@record[attribute]}"
-        end
-      end
+    def defined_zones
+      @defined_zones ||= Zone.all.map(&:name)
     end
-  end
-
-  def initialize(record)
-    @record = record
-  end
 
-  def put
-    typecast
-    transform
-    return { errors: 'no record provided' } if @record.blank?
-    validate
-    return { errors: errors } if errors.present?
-
-    save
-  end
-
-  def save
-    if exists?
-      update
-    else
-      insert
+    def expected_zones
+      config.fetch('zones')
     end
-    @record
-  end
-
-  def database
-    self.class.database
-  end
-
-  def dataset
-    self.class.dataset
-  end
-
-  def errors
-    @errors ||= []
-  end
-
-  def typecast
-    if database.database_type == :postgres
-      Array(self.class.type_column_map(database)['json']).each do |json_column|
-        if @record[json_column]
-          @record[json_column] = Sequel.pg_json(@record[json_column])
-        end
-      end
-    end
-  end
-
-  def transform
-  end
-
-  def validate
-    self.class.validations.each do |validation|
-      instance_eval(&validation)
-    end
-  end
-
-  def exists?
-    !dataset.select(1).where(id: @record[:id]).empty?
-  end
-
-  def insert
-    @record[:id] = dataset.insert(@record.except(:errors))
-  end
-
-  def update
-    dataset.where(id: @record[:id]).update(@record.except(:id, :errors))
   end
 end

data/lib/record_store/changeset.rb

@@ -0,0 +1,85 @@
+module RecordStore
+  class Changeset
+    class Change
+      attr_accessor :type, :record, :id
+
+      def initialize(type: nil, record: nil, id: nil)
+        @type, @record, @id = type, record, id
+      end
+
+      def self.addition(record)
+        new(type: :addition, record: record)
+      end
+
+      def self.removal(record)
+        new(type: :removal, record: record)
+      end
+
+      def self.update(id, record)
+        raise ArgumentError.new('id cannot be nil') if id.nil?
+        new(type: :update, record: record, id: id)
+      end
+
+      def removal?
+        type == :removal
+      end
+
+      def addition?
+        type == :addition
+      end
+
+      def update?
+        type == :update
+      end
+    end
+
+    attr_reader :current_records, :desired_records, :removals, :additions, :updates
+
+    def initialize(current_records: [], desired_records: [])
+      @current_records, @desired_records = Set.new(current_records), Set.new(desired_records)
+      @additions, @removals, @updates = [], [], []
+
+      build_changeset
+    end
+
+    def unchanged
+      current_records & desired_records
+    end
+
+    def changes
+      updates.to_a + removals.to_a + additions.to_a
+    end
+
+    def empty?
+      changes.empty?
+    end
+
+    private
+
+    def build_changeset
+      current_records_set = (current_records - unchanged).group_by(&:key)
+      desired_records_set = (desired_records - unchanged).group_by(&:key)
+      current_records_set.default_proc = desired_records_set.default_proc = Proc.new{Array.new}
+
+      record_keys = current_records_set.keys | desired_records_set.keys
+
+      diff = record_keys.flat_map do |key|
+        # the array being zipped over must be bigger then or equal to the other array for zip to work
+        buffer = [0, desired_records_set[key].size - current_records_set[key].size].max
+        temp_diff = current_records_set[key] + [nil] * buffer
+
+        temp_diff.zip(desired_records_set[key])
+      end
+
+      diff.each do |before_rr, after_rr|
+        if before_rr.nil?
+          @additions << Change.addition(after_rr)
+        elsif after_rr.nil?
+          @removals << Change.removal(before_rr)
+        else
+          @updates << Change.update(before_rr.id, after_rr)
+        end
+      end
+    end
+  end
+end

data/lib/record_store/cli.rb

@@ -0,0 +1,255 @@
+module RecordStore
+  class CLI < Thor
+    class_option :config, desc: 'Path to config.yml', aliases: '-c'
+
+    def initialize(*args)
+      super
+      RecordStore.config_path = options.fetch('config', "#{Dir.pwd}/config.yml")
+    end
+
+    desc 'thaw', 'Thaws all zones under management to allow manual edits'
+    def thaw
+      Zone.each do |_, zone|
+        zone.provider.thaw if zone.provider.thawable?
+      end
+    end
+
+    desc 'freeze', 'Freezes all zones under management to prevent manual edits'
+    def freeze
+      Zone.each do |_, zone|
+        zone.provider.freeze_zone if zone.provider.freezable?
+      end
+    end
+
+    desc 'list', 'Lists out records in YAML zonefiles'
+    def list
+      Zone.each do |name, zone|
+        puts "Zone: #{name}"
+        zone.records.each do |record|
+          record.log!
+        end
+      end
+    end
+
+    option :verbose, desc: 'Print records that haven\'t diverged', aliases: '-v', type: :boolean, default: false
+    desc 'diff', 'Displays the DNS differences between the zone files in this repo and production'
+    def diff
+      Zone.each do |name, zone|
+        puts "Zone: #{name}"
+
+        diff = zone.changeset
+
+        if !diff.additions.empty? || options.fetch('verbose')
+          puts "Add:"
+          diff.additions.map(&:record).each do |record|
+            puts " - #{record.to_s}"
+          end
+        end
+
+        if !diff.removals.empty? || options.fetch('verbose')
+          puts "Remove:"
+          diff.removals.map(&:record).each do |record|
+            puts " - #{record.to_s}"
+          end
+        end
+
+        if !diff.updates.empty? || options.fetch('verbose')
+          puts "Update:"
+          diff.updates.map(&:record).each do |record|
+            puts " - #{record.to_s}"
+          end
+        end
+
+        if options.fetch('verbose')
+          puts "Unchanged:"
+          diff.unchanged.each do |record|
+            puts " - #{record.to_s}"
+          end
+        end
+
+        puts "Empty diff" if diff.changes.empty?
+        puts '=' * 20
+      end
+    end
+
+    desc 'apply', 'Applies the DNS changes'
+    def apply
+      zones = zones_modified
+
+      if zones.empty?
+        puts "No changes to sync"
+        exit
+      end
+
+      zones.each do |zone|
+        abort "Attempted to apply invalid zone: #{zone.name}" unless zone.valid?
+        provider = zone.provider
+        provider.apply_changeset(zone.changeset)
+      end
+
+      puts "All zone changes deployed"
+    end
+
+    option :name, desc: 'Zone to download', aliases: '-n', type: :string, required: true
+    option :provider, desc: 'Provider in which this zone exists', aliases: '-p', type: :string
+    desc 'download', 'Downloads all records from zone and creates YAML zone definition in zones/ e.g. record-store download --name=shopify.io'
+    def download
+      name = options.fetch('name')
+      abort 'Please omit the period at the end of the zone' if name.ends_with?('.')
+      abort 'Zone with this name already exists in zones/' if File.exists?("#{RecordStore.zones_path}/#{name}.yml")
+
+      provider = options.fetch('provider', Provider.provider_for(name))
+      if provider.nil?
+        puts "Could not find valid provider from #{name} SOA record"
+        provider = ask("Please enter the provider in which #{name} exists")
+      else
+        puts "Identified #{provider} as the DNS provider"
+      end
+
+      puts "Downloading records for #{name}"
+      Zone.download(name, provider)
+      puts "Records have been downloaded & can be found in zones/#{name}.yml"
+    end
+
+    option :name, desc: 'Zone to sort', aliases: '-n', type: :string, required: true
+    desc 'sort', 'Sorts the zonefile alphabetically e.g. record-store sort --name=shopify.io'
+    def sort
+      name = options.fetch('name')
+      abort "Please omit the period at the end of the zone" if name.ends_with?('.')
+
+      yaml = YAML.load_file("#{RecordStore.zones_path}/#{name}.yml")
+      yaml.fetch(name).fetch('records').sort_by! { |r| [r.fetch('fqdn'), r.fetch('type'), r['nsdname'] || r['address']] }
+
+      File.write("#{RecordStore.zones_path}/#{name}.yml", yaml.deep_stringify_keys.to_yaml.gsub("---\n", ''))
+    end
+
+    desc 'secrets', 'Decrypts DynECT credentials'
+    def secrets
+      environment = begin
+        if ENV['PRODUCTION']
+          'production'
+        elsif ENV['CI']
+          'ci'
+        else
+          'dev'
+        end
+      end
+
+      secrets = `ejson decrypt #{RecordStore.secrets_path.sub(/\.json\z/, ".#{environment}.ejson")}`
+      if $?.exitstatus == 0
+        File.write(RecordStore.secrets_path, secrets)
+      else
+        abort secrets
+      end
+    end
+
+    desc 'assert_empty_diff', 'Asserts there is no divergence between DynECT & the zone files'
+    def assert_empty_diff
+      zones = zones_modified.map(&:name)
+
+      unless zones.empty?
+        abort "The following zones have diverged: #{zones.join(', ')}"
+      end
+    end
+
+    desc 'validate_records', 'Validates that all DNS records have valid definitions'
+    def validate_records
+      invalid_zones = []
+      Zone.each do |name, zone|
+        if !zone.valid?
+          invalid_zones << name
+
+          puts "#{name} definition is not valid:"
+          zone.errors.each do |field, msg|
+            puts " - #{field}: #{msg}"
+          end
+
+          invalid_records = zone.records.reject(&:valid?)
+          puts '  Invalid records' if invalid_records.size > 0
+
+          invalid_records.each do |record|
+            puts "    #{record.to_s}"
+            record.errors.each do |field, msg|
+              puts "      - #{field}: #{msg}"
+            end
+          end
+        end
+      end
+
+      if invalid_zones.size > 0
+        abort "The following zones were invalid: #{invalid_zones.join(', ')}"
+      else
+        puts "All records have valid definitions."
+      end
+    end
+
+    desc 'validate_change_size', "Validates no more then particular limit of DNS records are removed per zone at a time"
+    def validate_change_size
+      zones = zones_modified
+
+      unless zones.empty?
+        removals = zones.select do |zone|
+          zone.changeset.removals.size > MAXIMUM_REMOVALS
+        end
+
+        unless removals.empty?
+          abort "As a safety measure, you cannot remove more than #{MAXIMUM_REMOVALS} records at a time per zone. (zones failing this: #{removals.map(&:name).join(', ')})"
+        end
+      end
+    end
+
+    desc 'validate_all_present', 'Validates that all the zones that are expected are defined'
+    def validate_all_present
+      defined_zones = Set.new(RecordStore.defined_zones)
+      expected_zones = Set.new(RecordStore.expected_zones)
+
+      missing_zones = expected_zones - defined_zones
+      extra_zones = defined_zones - expected_zones
+
+      unless missing_zones.empty?
+        abort "The following zones are missing: #{missing_zones.to_a.join(', ')}"
+      end
+
+      unless extra_zones.empty?
+        abort "The following unexpected zones are defined: #{extra_zones.to_a.join(', ')}"
+      end
+    end
+
+    SKIP_CHECKS = 'SKIP_DEPLOY_VALIDATIONS'
+    desc 'validate_initial_state', "Validates state hasn't diverged since the last deploy"
+    def validate_initial_state
+      begin
+        assert_empty_diff
+        puts "Deploy will cause no changes, no need to validate initial state"
+      rescue SystemExit
+        if File.exists?(File.expand_path(SKIP_CHECKS, Dir.pwd))
+          puts "Found '#{SKIP_CHECKS}', skipping predeploy validations"
+        else
+          puts "Checkout git SHA #{ENV['LAST_DEPLOYED_SHA']}"
+          `git checkout #{ENV['LAST_DEPLOYED_SHA']}`
+          abort "Checkout of old commit failed" if $?.exitstatus != 0
+
+          `record-store assert_empty_diff`
+          abort "Dyn status has diverged!" if $?.exitstatus != 0
+
+          puts "Checkout git SHA #{ENV['REVISION']}"
+          `git checkout #{ENV['REVISION']}`
+          abort "Checkout of new commit failed" if $?.exitstatus != 0
+        end
+      end
+    end
+
+    private
+
+    def zones_modified
+      modified_zones, mutex = [], Mutex.new
+      Zone.all.map do |zone|
+        thread = Thread.new do
+          mutex.synchronize {modified_zones << zone} unless zone.unchanged?
+        end
+      end.each(&:join)
+
+      modified_zones
+    end
+  end
+end