getrepos 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 26c87a9d420ba019d3484fa10d85fb617acaa31f
+  data.tar.gz: 64de56514c2b8d83e0a08a41f72c742e174af176
+SHA512:
+  metadata.gz: d3e7deb628a44a5c5dcc71ef377e60ecd6f6b36edebbc52f4d8fe6f61652537048b416d14f510ed9695f4e2f294662d42a728fb5d456b5b4bdcbaeb41e731947
+  data.tar.gz: ef4d517245dacc05e85f4d5c158c9182134209cf94bb388d6a4bd767017b8225ecafe1960ff3ffd5c55014721ad0d3e4a4efb5655cfb07de1f0b96ad0db34b16

data/.gitignore

@@ -0,0 +1,6 @@
+.DS_Store
+/pkg
+/build
+/test/build
+/test/init.json
+/test/.roundup.*

data/README.md

@@ -0,0 +1,154 @@
+# getrepos
+
+### Installation
+
+```bash
+gem install getrepos
+```
+
+### Usage
+
+```bash
+getrepos -h/--help
+```
+
+```bash
+  NAME:
+
+    getrepos
+
+  DESCRIPTION:
+
+    Download, extract and filter content from repositories (Git/Tarball)
+
+  COMMANDS:
+
+    help    Display global or [command] help documentation
+    init    Create sample JSON
+    install Download and install archives
+
+  GLOBAL OPTIONS:
+
+    -h, --help
+        Display help documentation
+
+    -v, --version
+        Display version information
+```
+
+```bash
+getrepos init
+```
+
+```bash
+  NAME:
+
+    init
+
+  SYNOPSIS:
+
+    getrepos init <file.json> [-f]
+
+  DESCRIPTION:
+
+    Create <file.json> with sample configuration
+
+  EXAMPLES:
+
+    # Generate repos.json
+    getrepos init repos.json
+
+    # Overwrite repos.json
+    getrepos init repos.json -f
+
+  OPTIONS:
+
+    -f, --force
+        Force file overwrite
+```
+
+```bash
+getrepos install
+```
+
+```bash
+  NAME:
+
+    install
+
+  SYNOPSIS:
+
+    getrepos install <file.json>
+
+  DESCRIPTION:
+
+    Download and install archives from URLs defined in <file.json>
+
+  EXAMPLES:
+
+    # Process URLs from repos.json
+    getrepos install repos.json
+```
+
+### Sample JSON config file
+
+```bash
+getrepos init repos.json
+```
+
+* repos.json with some Nginx modules
+
+```json
+{"repos": [
+  {"name": "nginx", "version": "1.9.12", "url": "http://nginx.org/download/nginx-1.9.12.tar.gz"},
+
+  {"name": "luajit", "version": "2.1.0-beta2", "url": "http://luajit.org/download/LuaJIT-2.1.0-beta2.tar.gz"},
+  {"name": "lua-cjson", "version": "2.1.0.3", "url": "https://github.com/openresty/lua-cjson.git"},
+  {"name": "hiredis", "version": "v0.13.3", "url": "https://github.com/redis/hiredis.git"},
+  {"name": "libmaxminddb", "version": "1.1.4", "url": "https://github.com/maxmind/libmaxminddb/releases/download/1.1.4/libmaxminddb-1.1.4.tar.gz"},
+
+  {"name": "lua-nginx", "version": "v0.10.2", "url": "https://github.com/openresty/lua-nginx-module.git"},
+  {"name": "lua-upstream-nginx", "version": "v0.05", "url": "https://github.com/openresty/lua-upstream-nginx-module.git"},
+  {"name": "redis2-nginx", "version": "v0.12", "url": "https://github.com/openresty/redis2-nginx-module.git"},
+  {"name": "set-misc-nginx", "version": "v0.30", "url": "https://github.com/openresty/set-misc-nginx-module.git"},
+  {"name": "ngx_cache_purge", "version": "2.3", "url": "https://github.com/FRiCKLE/ngx_cache_purge/archive/2.3.tar.gz"},
+  {"name": "nginx-push-stream", "version": "0.5.1", "url": "https://github.com/wandenberg/nginx-push-stream-module.git"},
+  {"name": "ngx_http_auth_request", "version": "662785733552", "url": "http://mdounin.ru/hg/ngx_http_auth_request_module/archive/tip.tar.gz", "path": "ngx_http_auth_request_module-662785733552"},
+
+  {"name": "lua-resty-core", "version": "v0.1.5", "url": "https://github.com/openresty/lua-resty-core.git"},
+  {"name": "lua-resty-dns", "version": "v0.15", "url": "https://github.com/openresty/lua-resty-dns.git"},
+  {"name": "lua-resty-redis", "version": "v0.22", "url": "https://github.com/openresty/lua-resty-redis.git"},
+  {"name": "lua-resty-upstream-healthcheck", "version": "v0.04", "url": "https://github.com/openresty/lua-resty-upstream-healthcheck.git"},
+
+  {"name": "log4j", "version": "1.2.17", "url": "https://repo1.maven.org/maven2/log4j/log4j/1.2.17/log4j-1.2.17.jar"}
+]}
+```
+
+* Result after installation
+
+```bash
+getrepos install repos.json
+tree -L 2 build
+
+build
+└── repos
+    ├── hiredis-v0.13.3
+    ├── libmaxminddb-1.1.4
+    ├── log4j-1.2.17
+    ├── lua-cjson-2.1.0.3
+    ├── lua-nginx-v0.10.2
+    ├── lua-resty-core-v0.1.5
+    ├── lua-resty-dns-v0.15
+    ├── lua-resty-redis-v0.22
+    ├── lua-resty-upstream-healthcheck-v0.04
+    ├── lua-upstream-nginx-v0.05
+    ├── luajit-2.1.0-beta2
+    ├── nginx-1.9.12
+    ├── nginx-aaa-v0.9.1
+    ├── nginx-push-stream-0.5.1
+    ├── ngx_cache_purge-2.3
+    ├── ngx_http_auth_request-662785733552
+    ├── pcre-8.38
+    ├── redis2-nginx-v0.12
+    └── set-misc-nginx-v0.30
+```

data/Rakefile

@@ -0,0 +1,16 @@
+require 'bundler/gem_tasks'
+
+# Default
+task :default => :help
+
+# Help
+desc 'Show help'
+task :help do
+  system('rake -T')
+end
+
+# Test
+desc 'Run tests'
+task :test do
+  system('cd test && ./helper/roundup.sh')
+end

data/bin/getrepos

@@ -0,0 +1,68 @@
+#!/usr/bin/env ruby
+$LOAD_PATH.push File.expand_path('../../lib', __FILE__)
+
+require 'getrepos'
+require 'uri'
+require 'json'
+require 'colorize'
+require 'open-uri'
+require 'commander/import'
+
+program :version, GetRepos::VERSION
+program :description, 'Download, extract and filter content from repositories (Git/Tarball)'
+
+never_trace!
+
+command :init do |c|
+  c.syntax = 'getrepos init <file.json> [-f]'
+  c.summary = 'Create sample JSON'
+  c.description = 'Create <file.json> with sample configuration'
+  c.example 'Generate repos.json', 'getrepos init repos.json'
+  c.example 'Overwrite repos.json', 'getrepos init repos.json -f'
+  c.option '-f', '--force', 'Force file overwrite'
+
+  c.action do |args, options|
+    if args.length != 1
+      puts 'Wrong number of arguments'.light_red
+      command(:help).run(:init)
+      exit 1
+    end
+
+    filename = args.first
+    if File.exists?(filename) && ! options.force
+      puts "File '#{filename}' already exists (use -f to overwrite)".light_red
+      exit 1
+    end
+
+    GetRepos.init(filename)
+  end
+end
+
+command :install do |c|
+  c.syntax = 'getrepos install <file.json>'
+  c.summary = 'Download and install archives'
+  c.description = 'Download and install archives from URLs defined in <file.json>'
+  c.example 'Process URLs from repos.json', 'getrepos install repos.json'
+  c.action do |args, options|
+    if args.length != 1
+      puts 'Wrong number of arguments'.light_red
+      command(:help).run(:install)
+      exit 1
+    end
+
+    filename = args.first
+    if ! File.exists?(filename)
+      puts "File '#{filename}' not found".light_red
+      exit 1
+    end
+
+    GetRepos.install(filename)
+  end
+end
+
+if ARGV.empty?
+  global_option '-h', '--help', 'Display help documentation'
+  global_option '-v', '--version', 'Display version information'
+  command(:help).run
+  exit 1
+end

data/getrepos.gemspec

@@ -0,0 +1,29 @@
+# coding: utf-8
+$LOAD_PATH.push File.expand_path('../lib', __FILE__)
+
+require 'getrepos/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'getrepos'
+  spec.version       = GetRepos::VERSION
+  spec.authors       = ['Rogério Schneider']
+  spec.email         = ['stockrt@gmail.com']
+  spec.summary       = %q(Get content from repositories and URLs)
+  spec.description   = %q(Download, expand and filter content from Git and
+  Tarball repositories into local directories.  Works similar to a Berksfile or
+  a librarian-puppet catalog but is not coupled to any technology other than
+  keeping a directory filled with exploded archives from remote repositories
+  and URLs.  For example, it can be used to download Nginx modules before
+  compiling.)
+
+  spec.homepage      = 'https://github.com/stockrt/getrepos'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+
+  spec.add_runtime_dependency('commander', '~> 4.4')
+  spec.add_runtime_dependency('colorize', '~> 0.7')
+end

data/lib/getrepos/init.rb

@@ -0,0 +1,34 @@
+module GetRepos
+  module_function
+
+  def init(filename)
+    sample_json = %q({"repos": [
+  // Nginx
+  {"name": "nginx", "version": "1.9.12", "url": "http://nginx.org/download/nginx-1.9.12.tar.gz"},
+  // Deps
+  {"name": "luajit", "version": "2.1.0-beta2", "url": "http://luajit.org/download/LuaJIT-2.1.0-beta2.tar.gz"},
+  {"name": "lua-cjson", "version": "2.1.0.3", "url": "https://github.com/openresty/lua-cjson.git"},
+  {"name": "hiredis", "version": "v0.13.3", "url": "https://github.com/redis/hiredis.git"},
+  {"name": "libmaxminddb", "version": "1.1.4", "url": "https://github.com/maxmind/libmaxminddb/releases/download/1.1.4/libmaxminddb-1.1.4.tar.gz"},
+  // Native modules
+  {"name": "lua-nginx", "version": "v0.10.2", "url": "https://github.com/openresty/lua-nginx-module.git"},
+  {"name": "lua-upstream-nginx", "version": "v0.05", "url": "https://github.com/openresty/lua-upstream-nginx-module.git"},
+  {"name": "redis2-nginx", "version": "v0.12", "url": "https://github.com/openresty/redis2-nginx-module.git"},
+  {"name": "set-misc-nginx", "version": "v0.30", "url": "https://github.com/openresty/set-misc-nginx-module.git"},
+  {"name": "ngx_cache_purge", "version": "2.3", "url": "https://github.com/FRiCKLE/ngx_cache_purge/archive/2.3.tar.gz"},
+  {"name": "nginx-push-stream", "version": "0.5.1", "url": "https://github.com/wandenberg/nginx-push-stream-module.git"},
+  {"name": "ngx_http_auth_request", "version": "662785733552", "url": "http://mdounin.ru/hg/ngx_http_auth_request_module/archive/tip.tar.gz", "path": "ngx_http_auth_request_module-662785733552"},
+  // Lua modules
+  {"name": "lua-resty-core", "version": "v0.1.5", "url": "https://github.com/openresty/lua-resty-core.git"},
+  {"name": "lua-resty-dns", "version": "v0.15", "url": "https://github.com/openresty/lua-resty-dns.git"},
+  {"name": "lua-resty-redis", "version": "v0.22", "url": "https://github.com/openresty/lua-resty-redis.git"},
+  {"name": "lua-resty-upstream-healthcheck", "version": "v0.04", "url": "https://github.com/openresty/lua-resty-upstream-healthcheck.git"},
+  // Strange
+  {"name": "log4j", "version": "1.2.17", "url": "https://repo1.maven.org/maven2/log4j/log4j/1.2.17/log4j-1.2.17.jar"}
+]}
+)
+
+    File.write(filename, sample_json)
+    puts "Generated '#{filename}'".light_green
+  end
+end

data/lib/getrepos/install.rb

@@ -0,0 +1,115 @@
+require 'getrepos/util'
+
+module GetRepos
+  module_function
+
+  def install(filename)
+    FileUtils.mkdir_p('build/repos')
+    repos = JSON.parse(open(filename).read, symbolize_names: true)
+
+    unless is_valid_json_root?(repos)
+      puts "Invalid JSON '#{filename}' (missing required field)".light_red
+      exit 1
+    end
+
+    repos[:repos].each do |repo|
+      unless is_valid_json_repo?(repo)
+        puts "Invalid JSON entry '#{repo}' (missing required field)".light_red
+        exit 1
+      end
+
+      puts
+      puts 'Processing entry: '.light_cyan + repo.to_s
+      if is_git_url?(repo[:url])
+        # Git
+        install_git(repo)
+      else
+        # Archive
+        archive = URI(repo[:url]).path
+        if archive.end_with?('.tar.gz')
+          install_archive(repo, 'tar.gz')
+        elsif archive.end_with?('.tgz')
+          install_archive(repo, 'tgz')
+        elsif archive.end_with?('.tar.bz2')
+          install_archive(repo, 'tar.bz2')
+        elsif archive.end_with?('.tar.xz')
+          install_archive(repo, 'tar.xz')
+        elsif archive.end_with?('.zip')
+          install_archive(repo, 'zip')
+        elsif archive.end_with?('.jar')
+          install_archive(repo, 'jar')
+        elsif archive.end_with?('.war')
+          install_archive(repo, 'war')
+        elsif archive.end_with?('.ear')
+          install_archive(repo, 'ear')
+        else
+          puts "Unsupported file extension in URL '#{repo[:url]}'".light_red
+          exit 1
+        end
+      end
+    end
+  end
+
+  def install_git(repo)
+    dest_dir = "build/repos/#{repo[:name]}-#{repo[:version]}"
+    gitbare_dest_dir = "build/.gitbare/#{repo[:name]}"
+
+    # Clone
+    if File.exists?("#{gitbare_dest_dir}/HEAD")
+      puts "Updating cloned '#{repo[:url]}' inside '#{gitbare_dest_dir}'".light_green
+      ret = run_local("cd #{gitbare_dest_dir} && git fetch -q")
+      exit ret if ret != 0
+    else
+      puts "Cloning '#{repo[:url]}' to '#{gitbare_dest_dir}'".light_cyan
+      ret = run_local("git clone -q --mirror '#{repo[:url]}' '#{gitbare_dest_dir}'")
+      exit ret if ret != 0
+    end
+
+    # Extract
+    puts "Extracting and filtering '#{gitbare_dest_dir}' to '#{dest_dir}'".light_cyan
+    prep_dest_dir(dest_dir)
+    ret = run_local("cd #{gitbare_dest_dir} &&
+                    git archive '#{repo[:version]}' -- #{repo[:path]} | tar xf - -C '../../.tmprepo/'")
+    exit ret if ret != 0
+    FileUtils.mv("build/.tmprepo/#{repo[:path]}", dest_dir)
+  end
+
+  def install_archive(repo, ext)
+    dest_dir = "build/repos/#{repo[:name]}-#{repo[:version]}"
+    archive_dest_file = "build/.archive/#{repo[:name]}-#{repo[:version]}.#{ext}"
+
+    # Download
+    if File.exists?(archive_dest_file)
+      puts "Already downloaded '#{repo[:url]}' to '#{archive_dest_file}'".light_green
+    else
+      puts "Saving '#{repo[:url]}' to '#{archive_dest_file}'".light_cyan
+      FileUtils.mkdir_p(File.dirname(archive_dest_file))
+      IO.copy_stream(open(repo[:url]), archive_dest_file)
+    end
+
+    # Extract
+    puts "Extracting and filtering '#{archive_dest_file}' to '#{dest_dir}'".light_cyan
+    prep_dest_dir(dest_dir)
+    ret = 0
+    if ext == 'tar.gz' || ext == 'tgz'
+      ret = run_local("tar xzf '#{archive_dest_file}' -C 'build/.tmprepo/'")
+    elsif ext == 'tar.bz2'
+      ret = run_local("tar xjf '#{archive_dest_file}' -C 'build/.tmprepo/'")
+    elsif ext == 'tar.xz'
+      ret = run_local("tar xJf '#{archive_dest_file}' -C 'build/.tmprepo/'")
+    elsif ext == 'zip' || ext == 'jar' || ext == 'war' || ext == 'ear'
+      ret = run_local("unzip -o '#{archive_dest_file}' -d 'build/.tmprepo/'")
+    else
+      puts "Unsupported file extension '#{ext}'".light_red
+      exit 1
+    end
+    exit ret if ret != 0
+    # Was the archive expanded to the usual directory 'name-version'?
+    archive_usual_dir = "build/.tmprepo/#{repo[:name]}-#{repo[:version]}/#{repo[:path]}"
+    if File.directory?(archive_usual_dir)
+      FileUtils.mv(archive_usual_dir, dest_dir)
+    else
+      FileUtils.mv("build/.tmprepo/#{repo[:path]}", dest_dir)
+    end
+  end
+end

data/lib/getrepos/util.rb

@@ -0,0 +1,48 @@
+def is_valid_json_root?(data)
+  if data[:repos].nil?
+    puts 'Missing required key "repos"'.light_red
+    return false
+  else
+    return true
+  end
+end
+
+def is_valid_json_repo?(data)
+  return_flag = true
+
+  [:name, :version, :url].each do |key|
+    if data[key].nil?
+      puts "Missing required key \"#{key}\"".light_red
+      return_flag = false
+    end
+  end
+
+  return return_flag
+end
+
+def is_git_url?(url)
+  if File.extname(URI(url).path) == '.git'
+    return true
+  else
+    return false
+  end
+end
+
+def run_local(command)
+  response = %x(#{command})
+  exit_status = $?.to_i
+
+  if exit_status != 0
+    puts "Error: #{command} (exit status: #{exit_status})"
+    puts response
+  end
+
+  return exit_status
+end
+
+def prep_dest_dir(dest_dir)
+  FileUtils.rm_rf('build/.tmprepo')
+  FileUtils.mkdir_p('build/.tmprepo')
+  FileUtils.rm_rf(dest_dir)
+  FileUtils.mkdir_p(File.dirname(dest_dir))
+end

data/lib/getrepos/version.rb

@@ -0,0 +1,3 @@
+module GetRepos
+  VERSION = '0.1.0'
+end

data/lib/getrepos.rb

@@ -0,0 +1,3 @@
+require 'getrepos/init'
+require 'getrepos/install'
+require 'getrepos/version'

data/test/basic-usage-test.sh

@@ -0,0 +1,80 @@
+#!/bin/bash
+
+# http://bmizerany.github.io/roundup
+# https://github.com/bmizerany/roundup
+# wget https://raw.githubusercontent.com/bmizerany/roundup/master/roundup.sh
+
+describe "Basic usage and sample config JSON"
+
+validate_build_repo() {
+    # Nginx
+    test -f build/repos/nginx-1.9.12/configure
+    # Deps
+    test -f build/repos/luajit-2.1.0-beta2/Makefile
+    test -f build/repos/lua-cjson-2.1.0.3/Makefile
+    test -f build/repos/hiredis-v0.13.3/Makefile
+    test -f build/repos/libmaxminddb-1.1.4/configure
+    # Native modules
+    test -f build/repos/lua-nginx-v0.10.2/config
+    test -f build/repos/lua-upstream-nginx-v0.05/config
+    test -f build/repos/redis2-nginx-v0.12/config
+    test -f build/repos/set-misc-nginx-v0.30/config
+    test -f build/repos/ngx_cache_purge-2.3/config
+    test -f build/repos/nginx-push-stream-0.5.1/config
+    test -f build/repos/ngx_http_auth_request-662785733552/config
+    # Lua modules
+    test -d build/repos/lua-resty-core-v0.1.5/lib/resty
+    test -d build/repos/lua-resty-dns-v0.15/lib/resty
+    test -d build/repos/lua-resty-redis-v0.22/lib/resty
+    test -d build/repos/lua-resty-upstream-healthcheck-v0.04/lib/resty
+}
+
+before() {
+    rm -f init.json
+    rm -rf build
+}
+
+after() {
+    rm -f init.json
+    rm -rf build
+}
+
+it_display_help() {
+    ../bin/getrepos | grep "help documentation"
+    ../bin/getrepos init | grep "getrepos init"
+    ../bin/getrepos install | grep "getrepos install"
+}
+
+it_display_version() {
+    ../bin/getrepos -v | grep -E "getrepos [[:digit:]]"
+}
+
+it_init() {
+    ../bin/getrepos init init.json
+    test -f init.json
+}
+
+it_init_force() {
+    ../bin/getrepos init init.json
+    test -f init.json
+    ../bin/getrepos init init.json -f
+}
+
+it_init_is_valid_json() {
+    ../bin/getrepos init init.json
+    grep nginx-push-stream init.json
+    ./helper/valid_json.rb init.json
+}
+
+it_init_generate_valid_build_repo() {
+    ../bin/getrepos init init.json
+    ../bin/getrepos install init.json
+    validate_build_repo
+}
+
+it_install() {
+    ../bin/getrepos install repos-basic-usage.json
+    validate_build_repo
+}
+
+# EOF

data/test/check-test.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+
+describe "Check for absent and invalid values inside config JSON"
+
+before() {
+    rm -rf build
+}
+
+after() {
+    rm -rf build
+}
+
+it_check_absent_all() {
+    set +e
+    ret=$(../bin/getrepos install repos-check-absent-all.json)
+    set -e
+    echo "$ret" | grep 'Missing required key'
+    echo "$ret" | grep 'Invalid JSON entry.*missing required field'
+}
+
+it_check_absent_name() {
+    set +e
+    ret=$(../bin/getrepos install repos-check-absent-name.json)
+    set -e
+    echo "$ret" | grep 'Missing required key "name"'
+    echo "$ret" | grep 'Invalid JSON entry.*missing required field'
+}
+
+it_check_absent_repos() {
+    set +e
+    ret=$(../bin/getrepos install repos-check-absent-repos.json)
+    set -e
+    echo "$ret" | grep 'Missing required key "repos"'
+    echo "$ret" | grep 'Invalid JSON.*missing required field'
+}
+
+it_check_absent_url() {
+    set +e
+    ret=$(../bin/getrepos install repos-check-absent-url.json)
+    set -e
+    echo "$ret" | grep 'Missing required key "url"'
+    echo "$ret" | grep 'Invalid JSON entry.*missing required field'
+}
+
+it_check_absent_version() {
+    set +e
+    ret=$(../bin/getrepos install repos-check-absent-version.json)
+    set -e
+    echo "$ret" | grep 'Missing required key "version"'
+    echo "$ret" | grep 'Invalid JSON entry.*missing required field'
+}
+
+it_check_invalid_filetype() {
+    set +e
+    ret=$(../bin/getrepos install repos-check-invalid-filetype.json)
+    set -e
+    echo "$ret" | grep 'Unsupported file extension in URL'
+}
+
+# EOF