record_store 1.0.0 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 09372c0d086fbae7cc74ca55aee2a955e1a2f761
-  data.tar.gz: 5b7bfedc45cb04ac8170286a608df7c83adcc71f
+  metadata.gz: e96a230fae8c5a5a73e2c51a2097c5b14094f82d
+  data.tar.gz: 55c913c54e64b78454624b0844ebf1e3ca74fa36
 SHA512:
-  metadata.gz: 89ee26625e1b23999f383762e9ca785f52b9f89572f0b45b72b68e522139dea82dd52cf01f90a7bc62178aeda67d1e0538b07140f43ed139c13226284eead2f0
-  data.tar.gz: f3f01c6a5b40cb4714bc0f147070c6d2590731c688a60fe7ae1ca83366fdaedc205ed7f24f2025a9f26239dbafb4328a3923db021101c1ad17348e4ffc5693c5
+  metadata.gz: 512431b9f45815ee10a9b8b10228660b944b862a99be5e1ecb42705cab33fd069f8787971881e0a905722ddf20b127c121c091da61af93b93d1ee568b48f66f9
+  data.tar.gz: d0f22e77f0a914a0c3ee11cef077cc8bd1e31eb4ce04acc8e351e01c43cff7218831e9621350ab631bcd9e62f9c943e2b690c1f275242be05479567ca36c5b75

data/.gitignore

@@ -1,3 +1,4 @@
 ._*
 test/vcr_debug.log
 dev/
+pkg/

data/README.md

@@ -80,10 +80,11 @@ When running `bin/record-store apply`, a `Changeset` is generated by comparing t
 # Development
 
 To get started developing on Record Store, run `bin/setup`. This will create a development directory, `dev/`, that mimics what a production directory managing DNS records using Record Store would look like. Use it as a sandbox when developing Record Store.
+You can use `bin/console` to get play with the dev data, or you can `cd` into `dev/` and use `bin/record-store` to test out the CLI.
 
 ### Adding new Providers
 
-To add a new Provider, create a class inherriting `Provider` in [`lib/record_store/provider/`](lib/record_store/provider/). The [DynECT provider](lib/record_store/provider/dnsimple.rb) is good to use as a reference implementation.
+To add a new Provider, create a class inheriting `Provider` in [`lib/record_store/provider/`](lib/record_store/provider/). The [DynECT provider](lib/record_store/provider/dynect.rb) is good to use as a reference implementation.
 
 **Note**: _there's no need to wrap `Provider#apply_changeset` unless it's necessary to do something before/after making changes to a zone._
 

data/bin/console

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+
+$:.unshift File.expand_path('../../lib', __FILE__)
+require 'bundler/setup'
+require 'record_store'
+
+RecordStore.zones_path = File.expand_path('../../dev/zones', __FILE__)
+
+require 'irb'
+IRB.start

data/dev.yml

@@ -0,0 +1,10 @@
+# This file is for shopify internal development, feel free to ignore.
+name: recordstore
+
+up:
+  - ruby: 2.3.1
+  - bundler
+
+packages:
+  - git@github.com:Shopify/dev-shopify.git
+

data/lib/record_store/cli.rb

@@ -1,6 +1,7 @@
 module RecordStore
   class CLI < Thor
     class_option :config, desc: 'Path to config.yml', aliases: '-c'
+    FORMATS = %w[file directory]
 
     def initialize(*args)
       super
@@ -92,6 +93,7 @@ module RecordStore
 
     option :name, desc: 'Zone to download', aliases: '-n', type: :string, required: true
     option :provider, desc: 'Provider in which this zone exists', aliases: '-p', type: :string
+    option :format, desc: 'Format', aliases: '-f', type: :string, default: 'file', enum: FORMATS
     desc 'download', 'Downloads all records from zone and creates YAML zone definition in zones/ e.g. record-store download --name=shopify.io'
     def download
       name = options.fetch('name')
@@ -107,10 +109,26 @@ module RecordStore
       end
 
       puts "Downloading records for #{name}"
-      Zone.download(name, provider)
+      Zone.download(name, provider, format: options.fetch('format').to_sym)
       puts "Records have been downloaded & can be found in zones/#{name}.yml"
     end
 
+    option :name, desc: 'Zone to reformat', aliases: '-n', type: :string, required: false
+    option :format, desc: 'Format', aliases: '-f', type: :string, default: 'file', enum: FORMATS
+    desc 'reformat', 'Sorts and re-outputs the zone (or all zones) as specified format (file)'
+    def reformat
+      name = options['name']
+      zones = if name
+        [Zone.find(name)]
+      else
+        Zone.all
+      end
+      zones.each do |zone|
+        puts "Writing #{zone.name}"
+        zone.write(format: options.fetch('format').to_sym)
+      end
+    end
+
     option :name, desc: 'Zone to sort', aliases: '-n', type: :string, required: true
     desc 'sort', 'Sorts the zonefile alphabetically e.g. record-store sort --name=shopify.io'
     def sort

data/lib/record_store/version.rb

@@ -1,3 +1,3 @@
 module RecordStore
-  VERSION = '1.0.0'
+  VERSION = '2.0.0'
 end

data/lib/record_store/zone/config.rb

@@ -12,6 +12,13 @@ module RecordStore
         @provider = provider
       end
 
+      def to_hash
+        {
+          provider: provider,
+          ignore_patterns: ignore_patterns,
+        }
+      end
+
       private
 
       def validate_zone_config

data/lib/record_store/zone.rb

@@ -8,7 +8,7 @@ module RecordStore
       end
 
       def defined
-        @defined ||= yaml_files.inject({}) { |zones, file| zones.merge(load_yaml_file(file)) }
+        @defined ||= yaml_files.inject({}) { |zones, file| zones.merge(load_yml_zone_definition(file)) }
       end
 
       def [](name)
@@ -21,19 +21,77 @@ module RecordStore
 
       def find(name)
         return unless File.exists?(zone_path = "#{RecordStore.zones_path}/#{name}.yml")
-        Zone.from_yaml_definition(*YAML.load_file(zone_path).first)
+        load_yml_zone_definition(zone_path).first.last
+      end
+
+      def write(name, config:, records:, format: :file)
+        raise ArgumentError, "format must be :directory or :file" unless %i(file directory).include?(format)
+        name = name.chomp('.')
+        zone_file = "#{RecordStore.zones_path}/#{name}.yml"
+        zone = { name => { config: config.to_hash } }
+        records = records.map(&:to_hash).sort_by! {|r| [r.fetch(:fqdn), r.fetch(:type), r[:nsdname] || r[:address]]}
+
+        if format == :file
+          zone[name][:records] = records
+          write_yml_file(zone_file, zone.deep_stringify_keys)
+          remove_record_files(name)
+        else
+          write_yml_file(zone_file, zone.deep_stringify_keys)
+          remove_record_files(name)
+          write_record_files(name, records)
+        end
       end
 
       private
 
-      def load_yaml_file(filename)
+      def write_yml_file(filename, data)
+        lines = data.to_yaml.lines
+        lines.shift if lines.first == "---\n"
+        File.write(filename, lines.join)
+      end
+
+      def load_yml_zone_definition(filename)
         result = {}
+        dir = File.dirname(filename)
         YAML.load_file(filename).each do |name, definition|
+          definition['records'] ||= []
+          Dir["#{dir}/#{name}/*__*.yml"].each do |record_file|
+            definition['records'] += load_yml_record_definitions(name, record_file)
+          end
           result[name] = Zone.from_yaml_definition(name, definition)
         end
         result
       end
 
+      def load_yml_record_definitions(name, record_file)
+        type, domain = File.basename(record_file, '.yml').split('__')
+        Array.wrap(YAML.load_file(record_file)).map do |record_definition|
+          record_definition.merge(fqdn: "#{domain}.#{name}", type: type)
+        end
+      end
+
+      def remove_record_files(name)
+        dir = "#{RecordStore.zones_path}/#{name}"
+        File.unlink(*Dir["#{dir}/*"])
+        Dir.unlink(dir)
+      rescue Errno::ENOENT
+      end
+
+      def write_record_files(name, records)
+        dir = "#{RecordStore.zones_path}/#{name}"
+        Dir.mkdir(dir)
+        records.group_by { |record| [record.fetch(:fqdn), record.fetch(:type)] }.each do |(fqdn, type), grouped_records|
+          grouped_records.each do |record|
+            record.delete(:fqdn)
+            record.delete(:type)
+            record.deep_stringify_keys!
+          end
+          grouped_records = grouped_records.first if grouped_records.size == 1
+          domain = fqdn.chomp('.').chomp(name).chomp('.')
+          write_yml_file("#{dir}/#{type}__#{domain}.yml", grouped_records)
+        end
+      end
+
       def yaml_files
         Dir["#{RecordStore.zones_path}/*.yml"]
       end
@@ -59,19 +117,13 @@ module RecordStore
       new(name, definition.deep_symbolize_keys)
     end
 
-    def self.download(name, provider_name)
+    def self.download(name, provider_name, **write_options)
       dns = new(name, config: {provider: provider_name}).provider
       current_records = dns.retrieve_current_records
-
-      File.write("#{RecordStore.zones_path}/#{name}.yml", {
-        name => {
-          config: {
-            provider: provider_name,
-            ignore_patterns: [{type: "NS", fqdn: "#{name}."}],
-          },
-          records: current_records.map(&:to_hash).sort_by! {|r| [r.fetch(:fqdn), r.fetch(:type), r[:nsdname] || r[:address]]}
-        }
-      }.deep_stringify_keys.to_yaml.gsub("---\n", ''))
+      write(name, records: current_records, config: {
+        provider: provider_name,
+        ignore_patterns: [{type: "NS", fqdn: "#{name}."}],
+      }, **write_options)
     end
 
     def initialize(name, records: [], config: {})
@@ -106,6 +158,10 @@ module RecordStore
       Provider.const_get(config.provider).new(zone: name.gsub(/\.\z/, ''))
     end
 
+    def write(**write_options)
+      self.class.write(name, config: config, records: records, **write_options)
+    end
+
     private
 
     def build_records(records)

data/lib/record_store.rb

@@ -1,6 +1,7 @@
 require 'json'
 require 'yaml'
 require 'active_support'
+require 'active_support/core_ext/array/wrap'
 require 'active_support/core_ext/hash/keys'
 require 'active_support/core_ext/string'
 require 'active_model'

data/template/zones/dnsimple.example.com/A__a-record.yml

@@ -0,0 +1,2 @@
+ttl: 86400
+address: 10.10.10.42

data/template/zones/dnsimple.example.com/TXT__marco.yml

@@ -0,0 +1,2 @@
+ttl: 300
+txtdata: polo

data/template/zones/dnsimple.example.com/TXT__ping.yml

@@ -0,0 +1,2 @@
+ttl: 300
+txtdata: pong

data/template/zones/dnsimple.example.com.yml

@@ -2,36 +2,5 @@ dnsimple.example.com:
   config:
     provider: DNSimple
     ignore_patterns:
-      - type: NS
-        fqdn: dnsimple.example.com.
-
-  records:
     - type: NS
       fqdn: dnsimple.example.com.
-      ttl: 3600
-      nsdname: ns1.dnsimple.com.
-    - type: NS
-      fqdn: dnsimple.example.com.
-      ttl: 3600
-      nsdname: ns2.dnsimple.com.
-    - type: NS
-      fqdn: dnsimple.example.com.
-      ttl: 3600
-      nsdname: ns3.dnsimple.com.
-    - type: NS
-      fqdn: dnsimple.example.com.
-      ttl: 3600
-      nsdname: ns4.dnsimple.com.
-
-    - type: A
-      fqdn: a-record.dnsimple.example.com.
-      address: 10.10.10.42
-      ttl: 86400
-    - type: TXT
-      fqdn: ping.dnsimple.example.com.
-      txtdata: pong
-      ttl: 300
-    - type: TXT
-      fqdn: marco.dnsimple.example.com.
-      txtdata: polo
-      ttl: 300

data/template/zones/dynect.example.com.yml

@@ -2,36 +2,18 @@ dynect.example.com:
   config:
     provider: DynECT
     ignore_patterns:
-      - type: NS
-        fqdn: dynect.example.com.
-
-  records:
-    - type: NS
-      fqdn: dynect.example.com.
-      ttl: 86400
-      nsdname: ns1.p19.dynect.net.
-    - type: NS
-      fqdn: dynect.example.com.
-      ttl: 86400
-      nsdname: ns2.p19.dynect.net.
-    - type: NS
-      fqdn: dynect.example.com.
-      ttl: 86400
-      nsdname: ns3.p19.dynect.net.
     - type: NS
       fqdn: dynect.example.com.
-      ttl: 86400
-      nsdname: ns4.p19.dynect.net.
-
-    - type: A
-      fqdn: a-record.dynect.example.com.
-      address: 10.10.10.42
-      ttl: 86400
-    - type: TXT
-      fqdn: ping.dynect.example.com.
-      txtdata: pong
-      ttl: 300
-    - type: TXT
-      fqdn: marco.dynect.example.com.
-      txtdata: polo
-      ttl: 300
+  records:
+  - type: A
+    fqdn: a-record.dynect.example.com.
+    ttl: 86400
+    address: 10.10.10.42
+  - type: TXT
+    fqdn: marco.dynect.example.com.
+    ttl: 300
+    txtdata: polo
+  - type: TXT
+    fqdn: ping.dynect.example.com.
+    ttl: 300
+    txtdata: pong

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: record_store
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 2.0.0
 platform: ruby
 authors:
 - Willem van Bergen
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-07-12 00:00:00.000000000 Z
+date: 2016-08-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -209,10 +209,12 @@ files:
 - LICENSE
 - README.md
 - Rakefile
+- bin/console
 - bin/record-store
 - bin/setup
 - bin/test
 - circle.yml
+- dev.yml
 - lib/record_store.rb
 - lib/record_store/changeset.rb
 - lib/record_store/cli.rb
@@ -240,6 +242,9 @@ files:
 - template/config.yml
 - template/secrets.json
 - template/zones/dnsimple.example.com.yml
+- template/zones/dnsimple.example.com/A__a-record.yml
+- template/zones/dnsimple.example.com/TXT__marco.yml
+- template/zones/dnsimple.example.com/TXT__ping.yml
 - template/zones/dynect.example.com.yml
 homepage: https://github.com/Shopify/record_store
 licenses:
@@ -261,7 +266,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.5
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: Manage DNS using git