gitdis 0.0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: cbc3764ebe71bcc03b916fb6a29b1b02daaf34dd
+  data.tar.gz: 3dbd971bf31a576d535e176e573023fe362d383b
+SHA512:
+  metadata.gz: 6da95b9acf0be41a12e4c588169d19ab10e72d48f381f4337fe229e166263b79d414be8cb2144cef37d6a9668de6771355bdf1ca1e28be4268372db013a26421
+  data.tar.gz: 64f87929af5d569b59bf08d2d927855ee657508fed9bd149ab37a577852062dfc4c8d94dfb972914a40fd032243298b6f21600088effeec78273d0f478700861

data/README.md

@@ -0,0 +1,89 @@
+# gitdis
+
+## Install
+
+### Prerequisites
+
+* Ruby (>= 2.0)
+
+### Procedure
+
+`gem install gitdis`
+
+## Usage
+
+`gitdis path/to/config.yaml [options]`
+
+## Config
+
+### YAML Config
+
+5 options, 1 keymap, plus environment overrides:
+
+```
+redis-host: localhost
+redis-port: 6379
+redis-db: 0
+git-repo: ~/foo/bar/baz
+git-branch: quux
+keymap:
+  "redis:key:1": "path/to/file1"
+  "redis:key:2": "path/to/file2"
+
+environments:
+  yours:
+    redis-host: localhost
+    git-branch: mine
+  mine:
+    redis-host: localhost
+    git-branch: yours
+  qa:
+    redis-host: qa.big.com
+    git-branch: develop
+  prod:
+    redis-host: secret.big.net
+    git-branch: master
+```
+
+### Command Line Options
+
+Select your environment (optional).  Add final overrides (optional).
+
+```
+  Environment selection
+    -e, --environment  select within YAML[environments]
+  Redis overrides
+    -H, --redis-host   string
+    -p, --redis-port   number
+    -d, --redis-db     number
+  Git repo overrides
+    -r, --git-repo     path/to/repo_dir
+    -b, --git-branch   e.g. master
+  Other options
+    -D, --dump         Just dump Redis contents per YAML keymap
+    -h, --help
+```
+
+### Basic operation
+
+1. pull the latest changes from origin on the specified branch
+2. iterate over all the expected filenames, skipping any that are missing
+3. calculate md5 for all the filenames
+4. compare the md5 to what is in redis
+5. update redis if the md5s do not match
+
+### Redis update
+
+Assuming `foo:bar:baz` base key:
+
+```
+# redis.connect(redis_options)
+GET foo:bar:baz:md5 # assume md5 mismatch
+SET foo:bar:baz     # file contents
+SET foo:bar:baz:md5 # file contents md5
+INCR foo:bar:baz:version
+```
+
+### Execution
+
+This script is short-running and intended to be scheduled by e.g. `cron`

data/VERSION

@@ -0,0 +1 @@
+0.0.1.1

data/bin/gitdis

@@ -0,0 +1,88 @@
+#!/usr/bin/env ruby
+
+require 'yaml'
+require 'slop'
+require 'gitdis'
+
+opts = Slop.parse { |o|
+  o.banner = "USAGE: gitdis path/to/config.yaml [options]"
+  o.separator '  Environment selection'
+  o.string '-e', '--environment', 'select within YAML[environments]'
+  o.separator '  Redis overrides'
+  o.string  '-H', '--redis-host', 'string'
+  o.integer '-p', '--redis-port', 'number'
+  o.integer '-d', '--redis-db',   'number'
+  o.separator '  Git repo overrides'
+  o.string '-r', '--git-repo',   'path/to/repo_dir'
+  o.string '-b', '--git-branch', 'e.g. master'
+  o.separator '  Other options'
+  o.bool '-D', '--dump', 'Just dump Redis contents per YAML keymap'
+  o.on   '-h', '--help' do
+    puts o
+    exit
+  end
+}
+
+unless(yaml_file = opts.arguments.shift)
+  puts opts
+  puts "path/to/config.yaml is required"
+  exit 1
+end
+
+# load the default (required) config
+yaml = YAML.load_file(yaml_file)
+config = yaml.fetch('default')
+
+# apply any env-specific settings
+if opts[:environment]
+  env = opts[:environment].downcase
+  unless yaml.key?(env)
+    warn "no environment #{env} found in #{yaml_file}"
+    exit 1
+  end
+  config.merge!(yaml[env])
+end
+
+# make sure we have a keymap
+keymap = config.fetch('keymap')
+raise "keymap should be a hash/map" unless keymap.is_a?(Hash)
+
+# update config with any settings from opts
+# create redis_options
+redis_options = {}
+%w[git-repo git-branch redis-host redis-port redis-db].each { |item|
+  opt_value = opts[item.to_sym]
+  config[item] = opt_value if opt_value
+
+  if item.match /^redis-/      # set :host, :port, :db
+    cfg_value = config[item]
+    if cfg_value
+      # "redis-foo-bar" => :foo_bar
+      ropt = item.split('-').drop(1).join('_').to_sym
+      redis_options[ropt] = cfg_value
+    end
+  end
+  puts [item, config[item]].join(': ') if config[item]
+}
+puts "Redis options: #{redis_options}"
+
+# do things'n'stuff
+if opts.dump?
+  GitDis.dump(keymap.keys, redis_options)
+else
+  gd = GitDis.new config.fetch('git-repo')
+  gd.git_pull config.fetch('git-branch')
+  keymap.each { |key, fileglob|
+    result = gd.update(key, fileglob)
+    case result
+    when nil
+      puts "#{fileglob} not found"
+    when false
+      puts "#{fileglob} unchanged"
+    else
+      ver, md5 = *result
+      puts "#{fileglob} updated:"
+      puts "\tVersion: #{ver} (#{md5})"
+    end
+  }
+end

data/lib/gitdis.rb

@@ -0,0 +1,115 @@
+require 'open3'  # used for rudimentary git pull
+require 'digest' # md5sum
+require 'redis'
+
+class GitDis
+  # return Process::Status, stream through STDOUT and STDERR
+  def self.exec(cmd, bytes=1024)
+    Open3.popen3(cmd) { |sin, sout, serr, thr|
+      sin.close_write
+      while !sout.eof or !serr.eof
+        ready = IO.select [sout, serr]
+        if ready
+          ready[0].each { |f| # ready for reading
+            begin
+              (f == sout ? $stdout : $stderr).print f.read_nonblock(bytes)
+            rescue EOFError => e
+              # ok
+            end
+          }
+        end
+      end
+      thr.value # Process::Status
+    }
+  end
+
+  # raise on nonzero exit code
+  def self.exec!(cmd)
+    status = self.exec(cmd)
+    raise "`#{cmd}` #{status}" unless status.exitstatus == 0
+    0
+  end
+
+  # file contents, version, md5
+  def self.keyset(base_key)
+    [base_key, [base_key, 'version'].join(':'), [base_key, 'md5'].join(':')]
+  end
+
+  def self.dump(keys, redis_options)
+    redis = Redis.new(redis_options)
+    keys.each { |base_key|
+      self.keyset(base_key).each { |rkey|
+        val = redis.get(rkey)
+        if val and val.include?("\n")
+          val = "\n" << val.split("\n").map { |line| "\t#{line}" }.join("\n")
+        end
+        puts ["[#{rkey}]", val].join(' ')
+      }
+    }
+    redis.disconnect
+  end
+
+  attr_accessor :repo_dir, :redis
+
+  def initialize(repo_dir, redis_options = {})
+    @repo_dir = File.expand_path(repo_dir)
+    raise "#{@repo_dir} does not exist!" unless Dir.exist? @repo_dir
+    @redis = Redis.new(redis_options)
+  end
+
+  def git_pull(git_branch)
+    Dir.chdir @repo_dir do
+      if self.class.exec("git diff --quiet HEAD").exitstatus != 0
+        raise "please stash your local changes"
+      end
+      self.class.exec! "git checkout #{git_branch}"
+      self.class.exec! "git pull"
+    end
+    self
+  end
+
+  # quick false if calculated md5 == redis md5
+  # otherwise update contents and md5; increment version
+  def update_redis(base_key, file_contents)
+    md5 = Digest::MD5.hexdigest(file_contents)
+    fkey, vkey, mkey = self.class.keyset(base_key)
+    return false if @redis.get(mkey) == md5
+
+    @redis.set(fkey, file_contents)
+    @redis.set(mkey, md5)
+    ver = @redis.incr(vkey)
+    [ver, md5]
+  end
+
+  # e.g. update('foo:bar:baz', 'foo/bar/*.baz')
+  # return nil        # path does not exist
+  #        false      # no update needed
+  #        [ver, md5] # updated
+  def update(base_key, relpath)
+    # handle e.g. "foo/bar/*.yaml"
+    files = Dir.glob(File.join(@repo_dir, relpath))
+    case files.length
+    when 0 then nil
+    when 1 then self.update_redis(base_key, File.read(files.first))
+    else
+      puts "concatenating #{files.length} files"
+      result = ''
+      sep = "\n"
+      files.each { |fname|
+        s = File.read(fname)
+        if s and !s.empty?
+          # scan for carriage returns (Microsoft text format)
+          sep = "\r\n" if sep == "\n" and s.include?("\r")
+          s << sep if s.last != "\n"
+          result << s
+        # debugging
+        elsif s
+          puts "#{fname} is empty"
+        else
+          puts "File.read(#{fname}) returned false/nil"
+        end
+      }
+      self.update_redis(base_key, result.chomp(sep))
+    end
+  end
+end

metadata

@@ -0,0 +1,90 @@
+--- !ruby/object:Gem::Specification
+name: gitdis
+version: !ruby/object:Gem::Version
+  version: 0.0.1.1
+platform: ruby
+authors:
+- Rick Hull
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-05-28 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: slop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: buildar
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+description: Should you use this?  YES!
+email: 
+executables:
+- gitdis
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- VERSION
+- bin/gitdis
+- lib/gitdis.rb
+homepage: https://github.com/rickhull/gitdis
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: Pull from git, push to redis and beyond!
+test_files: []