talos 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6490791a8be853eea69f1bd18fdc18921ab76a62
-  data.tar.gz: 62cc19aa2882f7ec979f381058fdbb2a72fbc7a9
+  metadata.gz: a963d1d621605ff8c6bbc9fcd5a66ec02f63509c
+  data.tar.gz: a16ebee4fbc780bf4911e6df1eeab33701f35e45
 SHA512:
-  metadata.gz: b620929ce16fb6f872660a250b88f583a4b5ddd965a47b4b9d31372331f6a71c06b52cc0b6160b0677d5a6d1fcbca43e4c537cc3889791fd6a834831f8b18c51
-  data.tar.gz: db70975dbeac2c5a8461b9761cfbaca65db63a0c68d4f4089e8cf52525cd7dd7497b2b6adecc1bb82436da2e2eaaa410fedebb6f66ac9c409b63ea2e0440c5c3
+  metadata.gz: 32b2795e08276905aec04ee7918da2041a612424772df39c65fe86ae961bea50cc44a3a53ab76fba03e2eed92bb7cc8dc84962dd79a9c3d55374b87ef6e023b4
+  data.tar.gz: c77f95c6f9c96a8cc25ba202548dda2be9d0495407106a307f380ce5edf484b1feb141dbb294981023011c4dcd447edabdf196df100baff91d8783f4bdf56303

data/README.md

@@ -0,0 +1,177 @@
+Talos
+=====
+
+[![Gem Version](https://badge.fury.io/rb/talos.svg)](http://badge.fury.io/rb/talos)
+[![Build Status](https://travis-ci.org/spotify/talos.png?branch=master)](https://travis-ci.org/spotify/talos)
+
+Talos is a rack application which servers Hiera yaml files over HTTP.
+It authorizes clients based on the SSL certificates issued by the Puppet CA and returns only the files in the Hiera scope.
+
+Talos is used to store and distribute secrets via Hiera to the masterless puppet clients.
+
+How it works
+------------
+Talos listens for incoming HTTP requests and returns compressed hiera
+tree based on the client's SSL certificate.
+
+To determine the list of files to send, Talos matches the certificate
+common name against a list of regular expressions.
+
+Fetching the tree
+-----------------
+
+It's possible to run a cron task or create a wrapper around the puppet
+agent. Here's an example of the client-side code which uses local puppet SSL key
+to authenticate:
+
+```ruby
+require 'puppet'
+Puppet[:confdir] = '/etc/puppetlabs/puppet/'
+`/usr/bin/curl -s --fail -X GET -k https://talos.internal}/ \
+  --cert #{Puppet[:hostcert]} --key #{Puppet[:hostprivkey]} \
+  --data-urlencode pool=#{Facter.value(:pool)} > /etc/talos/tree.tar.gz`
+`/bin/tar xzf /etc/talos/tree.tar.gz -C /etc/talos/hiera_secrets`
+```
+
+In this example the client also passes `pool` variable which will
+be included in the Hiera scope if `unsafe_scopes` option is enabled.
+
+The received copy of the tree could be included in the local hiera config
+and used in the normal puppet runs.
+
+Configuration
+-------------
+Talos configuration is stored in `/etc/talos/talos.yaml`:
+
+```yaml
+scopes:
+  # lon-puppet-a1: site = lon, role = puppet, pool = a
+  - match: '(?<site>[[:alpha:]]+)-(?<role>[a-z0-9]+)-(?<pool>[[:alpha:]]+)'
+    facts:
+      environment: production
+  - match: 'cloud\.example\.com'
+    facts:
+      environment: testing
+
+unsafe_scopes: true
+```
+
+When receiving a request, Talos iterates over `scopes` list and matches
+the client certificate against the `match` blocks. If the match is
+successful, Talos does 2 things:
+
+1. Adds all the named captures from the regexp to the Hiera scope
+2. Adds all the `facts` to the Hiera scope
+
+Talos will iterate over all the regexps updating the
+Hiera scope, meaning that the later matches will override the existing
+scope on collision.
+
+If `unsafe_scopes` option is enabled, Talos will also add all the parameters
+passed by the client to the Hiera scope.
+
+Hiera
+-----
+You need to provide `/etc/talos/hiera.yaml` file to configure Hiera
+backend on the Talos server:
+
+```yaml
+---
+:backends:
+  - yaml
+:hierarchy:
+  - 'hiera-secrets/fqdn/%{fqdn}'
+  - 'hiera-secrets/role/%{role}/%{pod}/%{pool}'
+  - 'hiera-secrets/role/%{role}/%{pod}'
+  - 'hiera-secrets/role/%{role}'
+  - 'hiera-secrets/pod/%{pod}'
+  - 'hiera-secrets/common'
+:yaml:
+  :datadir: '/etc/puppet'
+:merge_behavior: :deeper
+```
+
+Talos will use the `datadir` option to search for YAML files and it
+will return only the files that match the Hiera scope of the clients.
+
+
+Installing
+----------
+
+First, install talos using rubygems:
+
+    $ gem install talos
+
+Create a separate user and Document Root for the Rack application:
+
+    $ useradd talos --system --create-home --home-dir /var/lib/talos
+    $ mkdir -p /var/lib/talos/public /var/lib/talos/tmp /etc/talos
+    $ chown -R talos:talos /var/lib/talos/ /etc/talos
+
+Then copy [config.ru](config.ru) to `/var/lib/talos/` directory.
+
+You also need to copy and adjust [hiera.yaml](spec/fixtures/hiera.yaml) and
+[talos.yaml](spec/fixtures/talos.yaml) configs in `/etc/talos` directory.
+
+### Hiera repository
+
+You need to have a copy of the hiera-secrets repository available on the
+talos server. Make sure it's located at the `datadir` specified in
+`/etc/talos/hiera.yaml`
+
+### Apache
+
+You can run Talos using Passenger or any other application server. Make
+sure you use Puppet SSL keys to validate the client certificates and to
+forward `SSL_CLIENT_S_DN_CN` header:
+
+```
+<VirtualHost *:443>
+  DocumentRoot "/var/lib/talos/public"
+
+  <Directory "/var/lib/talos/public">
+    Require all granted
+  </Directory>
+
+  SSLEngine on
+  SSLCertificateFile "/etc/puppetlabs/puppet/ssl/certs/talos.internal.pem"
+  SSLCertificateKeyFile "/etc/puppetlabs/puppet/ssl/private_keys/talos.internal.pem"
+  SSLCertificateChainFile "/etc/puppetlabs/puppet/ssl/certs/ca.pem"
+  SSLCACertificatePath "/etc/ssl/certs"
+  SSLCACertificateFile "/etc/puppetlabs/puppet/ssl/certs/ca.pem"
+  SSLCARevocationFile "/etc/puppetlabs/puppet/ssl/crl.pem"
+  SSLVerifyClient require
+  SSLOptions +StdEnvVars +FakeBasicAuth
+  RequestHeader set SSL_CLIENT_S_DN_CN "%{SSL_CLIENT_S_DN_CN}s"
+</VirtualHost>
+```
+
+Contributing
+------------
+1. Fork the project on github
+2. Create your feature branch
+3. Open a Pull Request
+
+This project adheres to the [Open Code of Conduct][code-of-conduct]. By
+participating, you are expected to honor this code.
+
+[code-of-conduct]:
+https://github.com/spotify/code-of-conduct/blob/master/code-of-conduct.md
+
+License
+-----------------
+```text
+Copyright 2013-2016 Spotify AB
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+```

data/config.ru

@@ -1,6 +1,6 @@
 require 'rubygems'
 # uncomment if you wish to run from source code
-libdir = File.join(File.dirname(__FILE__), 'lib')
-$LOAD_PATH.unshift(libdir) unless $LOAD_PATH.include?(libdir)
+# libdir = File.join(File.dirname(__FILE__), 'lib')
+# $LOAD_PATH.unshift(libdir) unless $LOAD_PATH.include?(libdir)
 require 'talos'
 run Talos

data/lib/talos.rb

@@ -1,4 +1,20 @@
 #!/usr/bin/env ruby
+#--
+# Copyright 2015 Spotify AB
+#
+# The contents of this file are licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+#++
+
 require 'sinatra/base'
 require 'json'
 require 'hiera'
@@ -8,91 +24,89 @@ require 'archive/tar/minitar'
 require 'pathname'
 include Archive::Tar
 
-def get_scope(fqdn, role = nil, pod = nil, pool = nil)
-  # FQDN from ssl
-  # role FQDN or client if unavailable
-  # pod - FQDN or client
-  # site - from pod
-  # spenvironment - FQDN
-  # pool - hostname or default to a
-  scope = { 'fqdn' => fqdn }
-  result = /(([[:alpha:]]+)\d*)-([a-z0-9]+)-([[:alpha:]])+/.match(fqdn)
-
-  if !result.nil?
-    scope['pod'] = result.captures[0]
-    scope['site'] = result.captures[1]
-    scope['role'] = result.captures[2]
-    scope['pool'] = result.captures[3]
-    scope['site'] = scope['pod'].tr('[0-9]', '')
-  else
-    # FQDN is not following our naming standard
-    scope['pod'] = pod
-    scope['role'] = role
-    scope['pool'] = pool
-    scope['site'] = scope['pod'].tr('[0-9]', '') unless pod.nil?
-  end
-
-  scope
-end
-
-
 class Talos < Sinatra::Base
-  def absolute_datadir
-    datadir = settings.hiera[:yaml][:datadir]
-    datadir = File.join(File.dirname(__FILE__), '..', datadir) if Pathname.new(datadir).relative?
+  def self.prepare_config(path)
+    set :talos, YAML.load_file(path)
+    settings.talos['scopes'].each do |scope_config|
+      begin
+        scope_config['regexp'] = Regexp.new(scope_config['match'])
+      rescue
+        fail "Invalid regexp: #{scope_config['match']}"
+      end
+    end
   end
 
   configure :development do
     require 'sinatra/reloader'
     register Sinatra::Reloader
     set :hiera, Hiera::Config::load(File.expand_path('spec/fixtures/hiera.yaml'))
+    prepare_config('spec/fixtures/talos.yaml')
     set :show_exceptions, false
   end
+
   configure :production do
     set :hiera, Hiera::Config::load(File.expand_path('/etc/talos/hiera.yaml'))
+    prepare_config('/etc/talos/talos.yaml')
+    warn("SECURITY WARNING: unsafe_scopes are enabled, SSL authentication bypass is possible") if settings.talos['unsafe_scopes']
   end
 
-  get '/' do
-    fqdn = settings.development? ? params[:fqdn] : request.env['HTTP_SSL_CLIENT_S_DN_CN']
+  def absolute_datadir
+    datadir = settings.hiera[:yaml][:datadir]
+    datadir = File.join(File.dirname(__FILE__), '..', datadir) if Pathname.new(datadir).relative?
+  end
 
-    # Get role/pod/pool from GET data(for hosts that don't follow our naming standard)
-    unless params[:role].nil?
-      role = params[:role].empty? ? nil : params[:role]
-    end
-    unless params[:pod].nil?
-      pod = params[:pod].empty? ? nil : params[:pod]
-    end
-    unless params[:pool].nil?
-      pool = params[:pool].empty? ? nil : params[:pool]
+  # Extracts scopes from FQDN using regexp with named captures
+  # Falls back to insecure arguments passed by the puppet agent
+  # (needed for the hosts not following naming convention)
+  def get_scope(fqdn)
+    scope = {'fqdn' => fqdn}
+    settings.talos['scopes'].each do | scope_config|
+      if m = fqdn.match(scope_config['regexp'])
+        scope.update(Hash[ m.names.zip( m.captures ) ])
+        scope.update(scope_config['facts'])
+      end
     end
 
-    scope = get_scope(fqdn, role, pod, pool)
-    files_to_pack = []
+    unsafe_scope = settings.talos['unsafe_scopes'] ? request.env['rack.request.query_hash'] : {}
+    scope.update(unsafe_scope)
+    # scope = {"pod"=>"lon3", "site"=>"lon", "role"=>"puppet", "pool"=>"a"}
+    scope
+  end
 
-    # Get all yaml files matching scope
+  def files_in_scope(scope)
+    files = []
     Hiera::Backend.datasources(scope, nil) do |source, yamlfile|
       yamlfile = Hiera::Backend.datafile(:yaml, scope, source, 'yaml') || next
       next unless File.exist?(yamlfile)
       # Strip path from filename
-      files_to_pack << yamlfile.gsub(settings.hiera[:yaml][:datadir] + '/', '')
+      files << yamlfile.gsub(settings.hiera[:yaml][:datadir] + '/', '')
     end
+    files
+  end
 
+  def compress_files(files)
     output = StringIO.new
     begin
       sgz = Zlib::GzipWriter.new(output)
       tar = Minitar::Output.new(sgz)
-
-      Dir.chdir(absolute_datadir) do
-        files_to_pack.each { |f| Minitar.pack_file(f, tar) }
-      end
+      Dir.chdir(absolute_datadir) { files.each { |f| Minitar.pack_file(f, tar) } }
     ensure
       tar.close
     end
+    output
+  end
+
+  get '/' do
+    fqdn = settings.development? ? params[:fqdn] : request.env['HTTP_SSL_CLIENT_S_DN_CN']
+    scope = get_scope(fqdn)
+    files_to_pack = files_in_scope(scope)
+    archive = compress_files(files_to_pack)
     content_type 'application/x-gzip'
-    output.string
+    archive.string
   end
 
   # Get the checksum the data folder symlink to
+  # Internal API
   get '/status' do
     begin
       File.readlink(absolute_datadir).split('.').last

data/spec/fixtures/hiera.yaml

@@ -4,15 +4,15 @@
 
 :hierarchy:
     - 'fqdn/%{fqdn}'
-    - 'role/%{role}/%{spenvironment}/%{pool}'
+    - 'role/%{role}/%{environment}/%{pool}'
     - 'role/%{role}/%{pool}'
-    - 'role/%{role}/%{spenvironment}'
+    - 'role/%{role}/%{environment}'
     - 'role/%{role}'
     - 'lsbdistcodename/%{lsbdistcodename}'
     - 'domain/%{domain}'
     - 'pod/%{pod}'
     - 'site/%{site}'
-    - 'spenvironment/%{spenvironment}'
+    - 'environment/%{environment}'
     - common
 
 :yaml:

data/spec/fixtures/master.3fa3fd97848a72ae539b75bccd6028cd1d4e92e3/role/foobar/testing.yaml

@@ -0,0 +1 @@
+foo::bar: baz

data/spec/fixtures/master.3fa3fd97848a72ae539b75bccd6028cd1d4e92e3/site/sjc.yaml

@@ -0,0 +1 @@
+foo::bar: baz

data/spec/fixtures/talos.yaml

@@ -0,0 +1,10 @@
+unsafe_scopes: true
+
+scopes:
+  # lon3-puppet-a1: pod = lon3, site = lon3, role = puppet, pool = a
+  - match: '(?<pod>(?<site>[[:alpha:]]+)\d*)-(?<role>[a-z0-9]+)-(?<pool>[[:alpha:]]+)\d+'
+    facts:
+      environment: production
+  - match: 'cloud\.example\.com'
+    facts:
+      environment: testing

data/spec/talos_spec.rb

@@ -5,26 +5,30 @@ describe 'talos' do
 
   def match_query_to_files(query, files)
     get query
-    last_response.should be_ok
-    file = Tempfile.new('spec')
-    file.write(last_response.body)
-    file.flush
-    output = `tar -tf #{file.path}`
-    file.unlink
-    files.each { |f| expect(output).to match(f) }
+    expect(last_response).to be_ok
+    Tempfile.open('spec') do |file|
+      file.write(last_response.body)
+      file.flush
+      files_in_archive = `tar -tf #{file.path}`.split
+      files.each { |f| expect(files.sort).to eq(files_in_archive.sort) }
+    end
   end
 
-  it 'should return common.yaml' do
+  it 'should detect scope and return YAML files' do
     { '/?fqdn=testing.int.sto.example.com' =>
-        ['common.yaml'],
+        %w(common.yaml),
       '/?role=puppet&pod=sto3&fqdn=sto3-puppet-a1.sto3.example.com' =>
-        ['common.yaml', 'role/puppet.yaml'],
+        %w(common.yaml role/puppet.yaml),
       '/?fqdn=sto3-puppet-a1.sto3.example.com' =>
-        ['common.yaml', 'role/puppet.yaml'],
+        %w(common.yaml role/puppet.yaml),
       '/?role=puppet&pod=sto3&fqdn=foo.bar' =>
         %w(common.yaml role/puppet.yaml fqdn/foo.bar.yaml),
       '/?fqdn=something.random&role=foobar&pool=z' =>
-        %w(common.yaml role/foobar/z.yaml)
+        %w(common.yaml role/foobar/z.yaml),
+      '/?fqdn=sjc1-puppet-a1' =>
+        %w(common.yaml role/puppet.yaml site/sjc.yaml),
+      '/?fqdn=sjc1-foobar-a1.cloud.example.com' =>
+        %w(common.yaml site/sjc.yaml role/foobar/testing.yaml),
     }.each do |query, files|
       match_query_to_files(query, files)
     end
@@ -32,7 +36,7 @@ describe 'talos' do
 
   it 'should resturn the checksum master symlink to' do
     get '/status'
-    last_response.should be_ok
-    last_response.body.should match('3fa3fd97848a72ae539b75bccd6028cd1d4e92e3')
+    expect(last_response).to be_ok
+    expect(last_response.body).to match('3fa3fd97848a72ae539b75bccd6028cd1d4e92e3')
   end
 end

data/talos.gemspec

@@ -1,10 +1,10 @@
 Gem::Specification.new do |s|
-  s.version       = '0.1.0'
+  s.version       = '0.1.1'
   s.name          = 'talos'
   s.authors       = ['Alexey Lapitsky', 'Johan Haals']
   s.email         = 'alexey@spotify.com'
-  s.summary     = %q{Distribute compressed hiera yaml files over HTTP}
-  s.description = %q{Serve hiera data over HTTP}
+  s.summary       = %q{Hiera secrets distribution over HTTP}
+  s.description   = %q{Distribute compressed hiera yaml files to authenticated puppet clients over HTTP}
   s.homepage      = 'https://github.com/spotify/talos'
   s.license       = 'Apache 2.0'
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: talos
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Alexey Lapitsky
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-12-17 00:00:00.000000000 Z
+date: 2016-01-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rack
@@ -109,7 +109,8 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '2.9'
-description: Serve hiera data over HTTP
+description: Distribute compressed hiera yaml files to authenticated puppet clients
+  over HTTP
 email: alexey@spotify.com
 executables: []
 extensions: []
@@ -119,14 +120,18 @@ files:
 - .rspec
 - Gemfile
 - LICENSE
+- README.md
 - Rakefile
 - config.ru
 - lib/talos.rb
 - spec/fixtures/hiera.yaml
 - spec/fixtures/master.3fa3fd97848a72ae539b75bccd6028cd1d4e92e3/common.yaml
 - spec/fixtures/master.3fa3fd97848a72ae539b75bccd6028cd1d4e92e3/fqdn/foo.bar.yaml
+- spec/fixtures/master.3fa3fd97848a72ae539b75bccd6028cd1d4e92e3/role/foobar/testing.yaml
 - spec/fixtures/master.3fa3fd97848a72ae539b75bccd6028cd1d4e92e3/role/foobar/z.yaml
 - spec/fixtures/master.3fa3fd97848a72ae539b75bccd6028cd1d4e92e3/role/puppet.yaml
+- spec/fixtures/master.3fa3fd97848a72ae539b75bccd6028cd1d4e92e3/site/sjc.yaml
+- spec/fixtures/talos.yaml
 - spec/spec_helper.rb
 - spec/talos_spec.rb
 - talos.gemspec
@@ -153,12 +158,15 @@ rubyforge_project:
 rubygems_version: 2.0.14
 signing_key: 
 specification_version: 4
-summary: Distribute compressed hiera yaml files over HTTP
+summary: Hiera secrets distribution over HTTP
 test_files:
 - spec/fixtures/hiera.yaml
 - spec/fixtures/master.3fa3fd97848a72ae539b75bccd6028cd1d4e92e3/common.yaml
 - spec/fixtures/master.3fa3fd97848a72ae539b75bccd6028cd1d4e92e3/fqdn/foo.bar.yaml
+- spec/fixtures/master.3fa3fd97848a72ae539b75bccd6028cd1d4e92e3/role/foobar/testing.yaml
 - spec/fixtures/master.3fa3fd97848a72ae539b75bccd6028cd1d4e92e3/role/foobar/z.yaml
 - spec/fixtures/master.3fa3fd97848a72ae539b75bccd6028cd1d4e92e3/role/puppet.yaml
+- spec/fixtures/master.3fa3fd97848a72ae539b75bccd6028cd1d4e92e3/site/sjc.yaml
+- spec/fixtures/talos.yaml
 - spec/spec_helper.rb
 - spec/talos_spec.rb