record_store 0.3.0 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d6555a4d5c8d65ea81e0341babdac09eba1c2df7
-  data.tar.gz: d1a73ddb5660a5b4e7a0dd157b632b5c6325dfb7
+  metadata.gz: 09372c0d086fbae7cc74ca55aee2a955e1a2f761
+  data.tar.gz: 5b7bfedc45cb04ac8170286a608df7c83adcc71f
 SHA512:
-  metadata.gz: eeb5548e87bd8d592d39254d18a6b19a2e56e69a8680320f240881dac8645503ed1561c39c7df4ae0ed97bb72d32e7348bfc81e3f796f26f0ae8b07014286c8e
-  data.tar.gz: 62f9e83f0a8cbc56b1923d0c8a4047712cff41eaf1d6bb8103f490f8c1307e9deb642f566100672a38bad87626bcc15d9a5038ef28caa550b08cf4d1dc8b810f
+  metadata.gz: 89ee26625e1b23999f383762e9ca785f52b9f89572f0b45b72b68e522139dea82dd52cf01f90a7bc62178aeda67d1e0538b07140f43ed139c13226284eead2f0
+  data.tar.gz: f3f01c6a5b40cb4714bc0f147070c6d2590731c688a60fe7ae1ca83366fdaedc205ed7f24f2025a9f26239dbafb4328a3923db021101c1ad17348e4ffc5693c5

data/.gitignore

@@ -1,9 +1,3 @@
-/.bundle/
-/.yardoc
-/Gemfile.lock
-/_yardoc/
-/coverage/
-/doc/
-/pkg/
-/spec/reports/
-/tmp/
+._*
+test/vcr_debug.log
+dev/

data/Gemfile

@@ -1,4 +1,21 @@
 source 'https://rubygems.org'
 
-# Specify your gem's dependencies in record_store.gemspec
-gemspec
+gem 'thor'
+gem 'activesupport', '~> 4.2'
+gem 'activemodel', '~> 4.2'
+gem 'ejson'
+
+gem 'fog'
+gem 'fog-json'
+gem 'fog-xml'
+gem 'fog-dynect'
+
+group :test do
+  gem 'mocha'
+  gem 'vcr'
+end
+
+group :development do
+  gem 'pry'
+  gem 'rake'
+end

data/Gemfile.lock

@@ -0,0 +1,178 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    CFPropertyList (2.3.2)
+    activemodel (4.2.2)
+      activesupport (= 4.2.2)
+      builder (~> 3.1)
+    activesupport (4.2.2)
+      i18n (~> 0.7)
+      json (~> 1.7, >= 1.7.7)
+      minitest (~> 5.1)
+      thread_safe (~> 0.3, >= 0.3.4)
+      tzinfo (~> 1.1)
+    builder (3.2.2)
+    coderay (1.1.0)
+    ejson (1.0.0)
+    excon (0.45.4)
+    fission (0.5.0)
+      CFPropertyList (~> 2.2)
+    fog (1.36.0)
+      fog-aliyun (>= 0.1.0)
+      fog-atmos
+      fog-aws (>= 0.6.0)
+      fog-brightbox (~> 0.4)
+      fog-core (~> 1.32)
+      fog-dynect (~> 0.0.2)
+      fog-ecloud (~> 0.1)
+      fog-google (<= 0.1.0)
+      fog-json
+      fog-local
+      fog-powerdns (>= 0.1.1)
+      fog-profitbricks
+      fog-radosgw (>= 0.0.2)
+      fog-riakcs
+      fog-sakuracloud (>= 0.0.4)
+      fog-serverlove
+      fog-softlayer
+      fog-storm_on_demand
+      fog-terremark
+      fog-vmfusion
+      fog-voxel
+      fog-xenserver
+      fog-xml (~> 0.1.1)
+      ipaddress (~> 0.5)
+      nokogiri (~> 1.5, >= 1.5.11)
+    fog-aliyun (0.1.0)
+      fog-core (~> 1.27)
+      fog-json (~> 1.0)
+      ipaddress (~> 0.8)
+      xml-simple (~> 1.1)
+    fog-atmos (0.1.0)
+      fog-core
+      fog-xml
+    fog-aws (0.7.6)
+      fog-core (~> 1.27)
+      fog-json (~> 1.0)
+      fog-xml (~> 0.1)
+      ipaddress (~> 0.8)
+    fog-brightbox (0.9.0)
+      fog-core (~> 1.22)
+      fog-json
+      inflecto (~> 0.0.2)
+    fog-core (1.32.1)
+      builder
+      excon (~> 0.45)
+      formatador (~> 0.2)
+      mime-types
+      net-scp (~> 1.1)
+      net-ssh (>= 2.1.3)
+    fog-dynect (0.0.2)
+      fog-core
+      fog-json
+      fog-xml
+    fog-ecloud (0.3.0)
+      fog-core
+      fog-xml
+    fog-google (0.1.0)
+      fog-core
+      fog-json
+      fog-xml
+    fog-json (1.0.2)
+      fog-core (~> 1.0)
+      multi_json (~> 1.10)
+    fog-local (0.2.1)
+      fog-core (~> 1.27)
+    fog-powerdns (0.1.1)
+      fog-core (~> 1.27)
+      fog-json (~> 1.0)
+      fog-xml (~> 0.1)
+    fog-profitbricks (0.0.5)
+      fog-core
+      fog-xml
+      nokogiri
+    fog-radosgw (0.0.4)
+      fog-core (>= 1.21.0)
+      fog-json
+      fog-xml (>= 0.0.1)
+    fog-riakcs (0.1.0)
+      fog-core
+      fog-json
+      fog-xml
+    fog-sakuracloud (1.4.0)
+      fog-core
+      fog-json
+    fog-serverlove (0.1.2)
+      fog-core
+      fog-json
+    fog-softlayer (1.0.2)
+      fog-core
+      fog-json
+    fog-storm_on_demand (0.1.1)
+      fog-core
+      fog-json
+    fog-terremark (0.1.0)
+      fog-core
+      fog-xml
+    fog-vmfusion (0.1.0)
+      fission
+      fog-core
+    fog-voxel (0.1.0)
+      fog-core
+      fog-xml
+    fog-xenserver (0.2.2)
+      fog-core
+      fog-xml
+    fog-xml (0.1.2)
+      fog-core
+      nokogiri (~> 1.5, >= 1.5.11)
+    formatador (0.2.5)
+    i18n (0.7.0)
+    inflecto (0.0.2)
+    ipaddress (0.8.0)
+    json (1.8.3)
+    metaclass (0.0.4)
+    method_source (0.8.2)
+    mime-types (2.6.1)
+    mini_portile (0.6.2)
+    minitest (5.9.0)
+    mocha (1.1.0)
+      metaclass (~> 0.0.1)
+    multi_json (1.11.2)
+    net-scp (1.2.1)
+      net-ssh (>= 2.6.5)
+    net-ssh (2.9.2)
+    nokogiri (1.6.6.2)
+      mini_portile (~> 0.6.0)
+    pry (0.10.1)
+      coderay (~> 1.1.0)
+      method_source (~> 0.8.1)
+      slop (~> 3.4)
+    rake (10.4.2)
+    slop (3.6.0)
+    thor (0.19.1)
+    thread_safe (0.3.5)
+    tzinfo (1.2.2)
+      thread_safe (~> 0.1)
+    vcr (2.9.3)
+    xml-simple (1.1.5)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  activemodel (~> 4.2)
+  activesupport (~> 4.2)
+  ejson
+  fog
+  fog-dynect
+  fog-json
+  fog-xml
+  mocha
+  pry
+  rake
+  thor
+  vcr
+
+BUNDLED WITH
+   1.11.2

data/{LICENSE.txt → LICENSE}

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2015 Paul Barry
+Copyright (c) 2016 Shopify
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
@@ -9,13 +9,13 @@ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 copies of the Software, and to permit persons to whom the Software is
 furnished to do so, subject to the following conditions:
 
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
 
 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -1,73 +1,160 @@
 # Record Store
 
-Store records in your database using Sequel
+Record Store is a tool to manage DNS through a git-based workflow.
 
-## Installation
+## Getting Started
 
-Add this line to your application's Gemfile:
-
-```ruby
-gem 'record_store'
 ```
+record-store apply                     # Applies the DNS changes
+record-store assert_empty_diff         # Asserts there is no divergence between DynECT & the zone files
+record-store diff                      # Displays the DNS differences between the zone files in this repo and production
+record-store download -n, --name=NAME  # Downloads all records from zone and creates YAML zone definition in zones/ e.g. record-store download --name=sho...
+record-store freeze                    # Freezes all zones under management to prevent manual edits
+record-store help [COMMAND]            # Describe available commands or one specific command
+record-store list                      # Lists out records in YAML zonefiles
+record-store secrets                   # Decrypts DynECT credentials
+record-store sort -n, --name=NAME      # Sorts the zonefile alphabetically e.g. record-store sort --name=shopify.io
+record-store thaw                      # Thaws all zones under management to allow manual edits
+record-store validate_all_present      # Validates that all the zones that are expected are defined
+record-store validate_change_size      # Validates no more then particular limit of DNS records are removed per zone at a time
+record-store validate_initial_state    # Validates state hasn't diverged since the last deploy
+record-store validate_records          # Validates that all DNS records have valid definitions
+```
+
+## Providers
+
+Below is the list of DNS providers supported by Record Store. PRs [adding more](#adding-new-providers) are welcome.
+
+### DNSimple
+
+Record Store uses DNSimple's [v1 API](https://developer.dnsimple.com/v1/). To use DNSimple, you'll need to add the primary user's `email` and `api_token` to `secrets.json`. There's currently no support for 2FA.
+
+### DynECT
+
+In order to use DynECT, you'll need to create a user that has the [correct read and write permissions](#dynect-permissions). Add the user's `username` & `password` (i.e. API password) as well as your DynECT `customer` name to `secrets.json`.
+
+#### Design
+
+The DynECT provider uses [DynECT's DNS API](https://help.dyn.com/rest-resources/) to sync the YAML zone files. DynECT uses an [update/publish cycle](https://help.dyn.com/understanding-works-api/) in their API which means no changes take place until POSTing to the publish endpoint. This allows us to handle all failures by discarding the changes we attempted to create.
+
+The DynECT zones managed by Record Store are frozen in DynECT (frozen zones cannot be changed); the deploy process will thaw them so it can make the necessary changes, and refreeze once the deploy process has completed.
+
+
+#### DynECT permissions
+
+The permissions required are broken into 2 groups:
+
+* READ: `RecordGet`, `ZoneGet`
+* WRITE: `RecordAdd`, `RecordDelete`, `ZonePublish`, `ZoneDiscardChangeset`, `ZoneFreeze`, `ZoneThaw`, `ZoneAddNode`,
+`ZoneRemoveNode`
+
+All CI validations only require READ permissions; deplyoing requires a user with READ and WRITE permissions.
+
+For a breakdown of what each permssion allows read through [DynECT's permissions guide](https://help.dyn.com/user-and-group-permissions/).
+
+----
+
+# Architecture
+
+All CLI commands are defined in [`lib/record_store/cli.rb`](lib/record_store/cli.rb) with [Thor](https://github.com/erikhuda/thor).
+
+### Zones and Records
+
+The `Zone` and `Record` models are representations of their DNS equivalents. Both have validations to ensure configurations are RFC compliant. These are specified using `ActiveModel::Validations`.
+
+Most CLI interactions are through the `Zone` model.
+
+### Providers
+
+In order to be provider agnostic, Record Store encapsulates all provider interactions in the `Provider` model and its children. A provider is initialized for each `zone`.
 
-And then execute:
 
-    $ bundle
+### Changeset
 
-Or install it yourself as:
+Changesets are how Record Store knows what updates to make. A `Changeset` is generated by comparing the current records in a zone with the desired final state. A `Changeset` is composed of one or more `Changeset::Change`. Each `Change` is either an `addition`, `removal`, or `update`. Since the ID of records aren't specified in zone files, FQDNs are used to dedup when records can be updated or when new ones need to be created.
 
-    $ gem install record_store
+When running `bin/record-store apply`, a `Changeset` is generated by comparing the current records in a zone's YAML file with the records the provider defines. A zone's YAML file is always considered the primary source of truth.
 
-## Usage
+----
 
-If you have an application where you are working with records as hashes and you would like store them in your database using Sequel, this gem can help make that happen. To use Record Store, you should subclass the `RecordStore` class at least once in your application. If for example you already have a reference to a Sequel database stored in a constant called `DB`, then you would do something lke this:
+# Development
 
+To get started developing on Record Store, run `bin/setup`. This will create a development directory, `dev/`, that mimics what a production directory managing DNS records using Record Store would look like. Use it as a sandbox when developing Record Store.
+
+### Adding new Providers
+
+To add a new Provider, create a class inherriting `Provider` in [`lib/record_store/provider/`](lib/record_store/provider/). The [DynECT provider](lib/record_store/provider/dnsimple.rb) is good to use as a reference implementation.
+
+**Note**: _there's no need to wrap `Provider#apply_changeset` unless it's necessary to do something before/after making changes to a zone._
+
+Provider API interactions are tested with [VCR](https://github.com/vcr/vcr). To generate the fixtures, update [`test/dummy/secrets.json`](test/dummy/secrets.json) with valid credentials, run the test suite, and change the values back to stub credentials.
+
+**Important**: _be sure to [filter sensitive data](https://github.com/Shopify/record_store/blob/1ec0d1410cf8bedf79bc63e8e4cdc7cdb0f1019b/test/test_helper.rb#L23-L56) from the fixtures or you're going to have a bad time._
+
+Outline of [`Provider`](lib/record_store/provider.rb):
 ```ruby
-class Store < RecordStore
-  def self.database
-    DB
+class Provider
+  # Creates a new record to the zone. It is expected this call modifies external state.
+  #
+  # Arguments:
+  # record - a kind of `Record`
+  def add(record)
   end
-end
-```
 
-Then for each table you want to store records in, you subclass that class:
+  # Deletes an existing record from the zone. It is expected this call modifies external state.
+  #
+  # Arguments:
+  # record - a kind of `Record`
+  def remove(record)
+  end
 
-```ruby
-class UserStore < Store
-end
-```
+  # Updates an existing record in the zone. It is expected this call modifies external state.
+  #
+  # Arguments:
+  # id - provider specific ID of record to update
+  # record - a kind of `Record` which the record with `id` should be updated to
+  def update(id, record)
+  end
 
-You can now store records in the `users` table like this:
+  # Downloads all the records from the provider.
+  #
+  # Returns: an array of `Record` for each record in the provider's zone
+  def retrieve_current_records
+  end
 
-```ruby
-UserStore << { first_name: 'Paul', last_name: 'Barry' }
-```
+  # Returns an array of the zones managed by provider as strings
+  def zones
+  end
 
-You can also add validations to your store, like this:
+  ######## NOTE ########
+  # The following methods only need to be implemented if the provider supports the ability to
+  # lock/unlock changes to zones.
+  ######################
 
-```ruby
-class UserStore < Store
-  required_attributes :first_name, :last_name
+  # Lock the ability to make any changes to the zone without unlocking it first. It is expected
+  # this call modifies external state.
+  def freeze_zone
+  end
+
+  # Unlocks the zone to allow making changes (see `Provider#freeze_zone`).
+  def thaw
+  end
 end
 ```
 
-If you try to store a record without values for those attributes, you will get an error:
+#### Provider-Specific Records
 
-```ruby
-UserStore << {}
- # => { errors: ["First Name is required", "Last Name is required"]}
-```
+For provider-specific records (e.g. `ALIAS`), create the record model in `lib/record_store/record` as any other record. In the provider, extend `self.record_types` and append the custom record types to the `Set` returned by `Provider.record_types` (e.g. [`DNSimple.record_types`](https://github.com/Shopify/record_store/blob/1ec0d1410cf8bedf79bc63e8e4cdc7cdb0f1019b/lib/record_store/provider/dnsimple.rb#L5-L7)).
+
+#### Secrets
+
+When adding a new provider, be sure to update the `secrets.json` in [`template/secrets.json`](template/secrets.json) and [`test/dummy/secrets.json`](test/dummy/secrets.json) with the new provider and required fields for the API to work.
 
-## Development
 
-After checking out the repo, run `bin/setup` to install dependencies. Then, run `bin/console` for an interactive prompt that will allow you to experiment.
+### Test Changes on Providers
 
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release` to create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+In order to test changes on providers, you're going to need to update `dev/secrets.json` with credentials. **Note**: make sure the credentials are for test zone(s) as the changes specified in the directory **will be applied**.
 
-## Contributing
+# Acknowledgements
 
-1. Fork it ( https://github.com/[my-github-username]/record_store/fork )
-2. Create your feature branch (`git checkout -b my-new-feature`)
-3. Commit your changes (`git commit -am 'Add some feature'`)
-4. Push to the branch (`git push origin my-new-feature`)
-5. Create a new Pull Request
+Big thanks to [@pjb3](https://github.com/pjb3) for graciously letting us use the `record_store` gem namespace.