datapimp 0.0.1 → 1.0.0

data/lib/datapimp/configuration.rb

@@ -0,0 +1,210 @@
+require 'singleton'
+require 'json'
+
+module Datapimp
+  class Configuration
+    include Singleton
+
+    DefaultSettings = {
+      manifest_filename: "datapimp.json",
+
+      github_username: '',
+      github_organization: '',
+      github_app_key: '',
+      github_app_secret: '',
+      github_access_token: '',
+
+      dnsimple_api_token: '',
+      dnsimple_username: '',
+
+      dropbox_app_key: '',
+      dropbox_app_secret: '',
+      dropbox_app_type: 'sandbox',
+      dropbox_client_token: '',
+      dropbox_client_secret: '',
+
+      aws_access_key_id: '',
+      aws_secret_access_key: '',
+
+      google_client_id: '',
+      google_client_secret: '',
+      google_refresh_token: '',
+      google_access_token: ''
+    }
+
+    def self.method_missing(meth, *args, &block)
+      if instance.respond_to?(meth)
+        return instance.send meth, *args, &block
+      end
+
+      nil
+    end
+
+    def method_missing(meth, *args, &block)
+      if current.key?(meth.to_s)
+        return current.fetch(meth)
+      end
+
+      super
+    end
+
+    def initialize!(fresh=false)
+      return if home_config_path.exist? && !fresh
+
+      FileUtils.mkdir_p home_config_path.dirname
+
+      home_config_path.open("w+") do |fh|
+        fh.write(DefaultSettings.to_json)
+      end
+    end
+
+    def manifest_filename
+      "datapimp.json"
+    end
+
+    def dnsimple_setup?
+      dnsimple_api_token.to_s.length > 0 && dnsimple_username.to_s.length > 0
+    end
+
+    def dropbox_setup?
+      dropbox_app_key.to_s.length > 0 && dropbox_app_secret.to_s.length > 0
+    end
+
+    def google_setup?
+      google_client_secret.to_s.length > 0 && google_client_id.to_s.length > 0
+    end
+
+    def amazon_setup?
+      aws_access_key_id.to_s.length > 0 && aws_secret_access_key.to_s.length > 0
+    end
+
+    def show
+      current.each do |p|
+        key, value = p
+
+        unless key == 'sites_directory'
+          puts "#{key}: #{ value.inspect }"
+        end
+      end
+    end
+
+    def primary_config
+      cwd_config_path.exist? ? cwd_config : home_config
+    end
+
+    def get(setting)
+      setting = setting.to_s.downcase
+      primary_config[setting]
+    end
+
+    def set(setting, value, persist = true, options={})
+      setting = setting.to_s.downcase
+      primary_config[setting] = value
+      save! if persist == true
+      value
+    end
+
+    def apply_all(options={})
+      current.merge!(options)
+    end
+
+    def unset(setting, persist = true)
+      primary_config.delete(setting)
+      save! if persist == true
+    end
+
+    def defaults
+      DefaultSettings.dup
+    end
+
+    def current(using_environment = true)
+      @current ||= calculate_config(using_environment)
+    end
+
+    def calculate_config(using_environment = true)
+      @current = defaults.merge(home_config.merge(cwd_config.merge(applied_config))).to_mash
+
+      (defaults.keys + home_config.keys + cwd_config.keys).uniq.each do |key|
+        upper = key.to_s.upcase
+        if ENV[upper]
+          @current[key] = ENV[upper]
+        end
+      end if using_environment
+
+      @current
+    end
+
+    def apply_config(hash={})
+      applied_config.merge!(hash)
+      current.merge(applied_config)
+    end
+
+    def apply_config_from_path(path)
+      path = Pathname(path)
+      parsed = JSON.parse(path.read) rescue {}
+      applied_config.merge!(parsed)
+      nil
+    end
+
+    def save!
+      save_home_config
+      save_cwd_config
+      @current = nil
+      true
+    end
+
+    def save_cwd_config
+      return nil unless cwd_config_path.exist?
+
+      File.open(cwd_config_path, 'w+') do |fh|
+        fh.write JSON.generate(cwd_config.to_hash)
+      end
+    end
+
+    def save_home_config
+      File.open(home_config_path, 'w+') do |fh|
+        fh.write JSON.generate(home_config.to_hash)
+      end
+    end
+
+    # Applied config is configuration values passed in context
+    # usually from the cli, but also in the unit tests
+    def applied_config
+      @applied_config ||= {}
+    end
+
+    def cwd_config
+      @cwd_config ||= begin
+        (cwd_config_path.exist? rescue false) ? JSON.parse(cwd_config_path.read) : {}
+      rescue
+        {}
+      end
+    end
+
+    def home_config
+      initialize! unless home_config_path.exist?
+
+      @home_config ||= begin
+        (home_config_path.exist? rescue false) ? JSON.parse(home_config_path.read) : {}
+      rescue
+        {}
+      end
+    end
+
+    def home_config_path= value
+      @home_config_path = Pathname(value)
+    end
+
+    def home_config_path
+      @home_config_path || Pathname(ENV['HOME']).join(".datapimp", manifest_filename)
+    end
+
+    def cwd_config_path= value
+      @cwd_config_path = Pathname(value)
+    end
+
+    def cwd_config_path
+      @cwd_config_path || Pathname(Datapimp.pwd).join(manifest_filename)
+    end
+  end
+end

data/lib/datapimp/core_ext.rb

@@ -0,0 +1,5 @@
+class Hash
+  def to_mash
+    Hashie::Mash.new(self)
+  end
+end

data/lib/datapimp/data_sources/dropbox.rb

@@ -0,0 +1,8 @@
+# The `Datapimp::Sources::Dropbox` class represents a
+# file on the Dropbox file sharing system (usually an excel file)
+# that will get turned into a JSON array of objects
+module Datapimp::Sources
+  class Dropbox < OpenStruct
+
+  end
+end

data/lib/datapimp/data_sources/excel.rb

@@ -0,0 +1,5 @@
+module Datapimp::Sources
+  class Excel < OpenStruct
+
+  end
+end

data/lib/datapimp/data_sources/github.rb

@@ -0,0 +1,5 @@
+module Datapimp::Sources
+  class Github < OpenStruct
+
+  end
+end

data/lib/datapimp/data_sources/google.rb

@@ -0,0 +1,5 @@
+module Datapimp::Sources
+  class Google < OpenStruct
+
+  end
+end

data/lib/datapimp/data_sources/json.rb

@@ -0,0 +1,5 @@
+module Datapimp::Sources
+  class Json < OpenStruct
+
+  end
+end

data/lib/datapimp/data_sources/nokogiri.rb

@@ -0,0 +1,5 @@
+module Datapimp::Sources
+  class Nokogiri < OpenStruct
+
+  end
+end

data/lib/datapimp/data_sources.rb

@@ -0,0 +1,10 @@
+# The `Datapimp::Dataources` module houses the various
+# types of remote data stores we are reading and converting into
+# a JSON array of objects that gets cached on our filesystem.
+module Datapimp
+  module DataSources
+
+  end
+end
+
+Dir[Datapimp.lib.join("datapimp/data_sources/**/*.rb")].each {|f| require(f) }

data/lib/datapimp/sync/dropbox_delta.rb

@@ -0,0 +1,67 @@
+module Datapimp
+  module Sync
+    class DropboxDelta
+
+      attr_accessor :client,
+                    :data,
+                    :cursor,
+                    :entries,
+                    :path_prefix
+
+      def initialize(client, cursor, path_prefix=nil)
+        @client = client
+        @cursor = cursor
+        @path_prefix = path_prefix
+      end
+
+      def processed!
+        # TODO
+        # Should update cursor
+      end
+
+      def entries
+        return @entries if @entries
+        fetch
+        @entries
+      end
+
+      def _dropbox_delta at=nil
+        at ||= cursor
+        response = client.delta(at, path_prefix)
+        self.cursor = response["cursor"]
+        response
+      end
+
+      def data
+        @data ||= fetch
+      end
+
+      def on_reset path_prefix, cursor
+      end
+
+      def fetch
+        return @response if @response
+
+        response = _dropbox_delta
+
+        if response["reset"] == true
+          on_reset(path_prefix, cursor)
+        end
+
+        self.entries = {}.to_mash
+
+        response["entries"].each do |entry|
+          path, meta = entry
+          self.entries[path] = meta
+        end
+
+        if response["has_more"] == true
+          # TODO Implement
+        end
+
+        @response = response
+      end
+
+    end
+  end
+end

data/lib/datapimp/sync/dropbox_folder.rb

@@ -0,0 +1,10 @@
+module Datapimp
+  class Sync::DropboxFolder < OpenStruct
+    class_attribute :default_path_prefix, :default_root
+
+
+    def push
+      Datapimp.dropbox(token: client_token, secret: client_secret)
+    end
+  end
+end

data/lib/datapimp/sync/google_drive_folder.rb

@@ -0,0 +1,5 @@
+module Datapimp
+  class Sync::GoogleDriveFolder < OpenStruct
+
+  end
+end

data/lib/datapimp/sync.rb

@@ -0,0 +1,30 @@
+# The `Datapimp::Sync` module will delegate to the underlying service layer
+# which we are pushing or pulling files and data from.  It will wrap the client
+# implementation we are using.
+module Datapimp
+  module Sync
+    def self.data_source_types
+      %w(dropbox amazon github google json excel nokogiri)
+    end
+
+    def self.amazon(options={})
+      require 'datapimp/clients/amazon'
+      Datapimp::Clients::Amazon.client(options)
+    end
+
+    def self.dropbox(options={})
+      require 'datapimp/clients/dropbox'
+      Datapimp::Clients::Dropbox.client(options)
+    end
+
+    def self.github(options={})
+      require 'datapimp/clients/github'
+      Datapimp::Clients::Github.client(options)
+    end
+
+    def self.google(options={})
+      require 'datapimp/clients/google'
+      Datapimp::Clients::Google.client(options)
+    end
+  end
+end

data/lib/datapimp/version.rb

@@ -1,3 +1,3 @@
 module Datapimp
-  VERSION = "0.0.1"
+  VERSION = "1.0.0"
 end

data/lib/datapimp.rb

@@ -1,5 +1,33 @@
-require "datapimp/version"
+require 'ostruct'
+require 'set'
+require 'pathname'
+require 'hashie'
+require 'datapimp/core_ext'
 
 module Datapimp
-  # Your code goes here...
+  def self.config
+    Datapimp::Configuration.instance
+  end
+
+  def self.pwd
+    Pathname(ENV.fetch('DATAPIMP_PWD') { Dir.pwd })
+  end
+
+  def self.lib
+    Pathname(File.dirname(__FILE__))
+  end
+
+  def self.method_missing(meth, *args, &block)
+    case
+    when %w(dropbox amazon github google).include?(meth.to_s)
+      Datapimp::Sync.send(meth, *args, &block)
+    else
+      super
+    end
+  end
 end
+
+require 'datapimp/version'
+require 'datapimp/configuration'
+require 'datapimp/data_sources'
+require 'datapimp/sync'

data/packaging/wrapper.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+export DATAPIMP_PWD=$PWD
+set -e
+
+TARGET_FILE=$0
+
+cd `dirname $TARGET_FILE`
+TARGET_FILE=`basename $TARGET_FILE`
+
+# Iterate down a (possible) chain of symlinks
+while [ -L "$TARGET_FILE" ]
+do
+    TARGET_FILE=`readlink $TARGET_FILE`
+    cd `dirname $TARGET_FILE`
+    TARGET_FILE=`basename $TARGET_FILE`
+done
+
+# Compute the canonicalized name by finding the physical path
+# for the directory we're in and appending the target file.
+PHYS_DIR=`pwd -P`
+RESULT=$PHYS_DIR/$TARGET_FILE
+
+# Figure out where this script is located.
+SELFDIR=$(dirname "$RESULT")
+
+# Tell Bundler where the Gemfile and gems are.
+export BUNDLE_GEMFILE="$SELFDIR/lib/app/Gemfile"
+unset BUNDLE_IGNORE_CONFIG
+
+# Run the actual app using the bundled Ruby interpreter.
+exec "$SELFDIR/lib/ruby/bin/ruby" -rbundler/setup "$SELFDIR/lib/app/bin/datapimp" "$@"

data/spec/spec_helper.rb

@@ -0,0 +1,29 @@
+require 'rack/test'
+require 'pry'
+require 'datapimp'
+
+Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each { |f| require f }
+
+module Datapimp
+  def self.spec_root
+    Pathname(File.dirname(__FILE__))
+  end
+
+  def self.dummy_path
+    spec_root.join("dummy")
+  end
+
+  def self.fixtures_path
+    spec_root.join("support","fixtures")
+  end
+end
+
+Skypager::Site.directory = {}
+
+RSpec.configure do |config|
+  config.mock_with :rspec
+  config.order = :random
+
+  config.include Rack::Test
+  config.include Requests::JsonHelpers, type: :request
+end

data/spec/support/test_helpers.rb

@@ -0,0 +1,7 @@
+module Requests
+  module JsonHelpers
+    def json
+      @json ||= JSON.parse(response.body)
+    end
+  end
+end

data/tasks/distribution/configuration.rb

@@ -0,0 +1,15 @@
+module Distribution
+  class << self
+    attr_accessor :configuration
+  end
+
+  def self.configure
+    self.configuration ||= Configuration.new
+    yield configuration
+  end
+
+  class Configuration
+    attr_accessor :package_name, :packaging_dir, :version, :rb_version,
+                  :native_extensions
+  end
+end

data/tasks/distribution/executable.rb

@@ -0,0 +1,28 @@
+Dir[File.join(Dir.pwd, 'tasks', '**', '*.rb')].each { |f| require f }
+
+module Distribution
+  class Executable
+    include PackageHelpers
+    extend Forwardable
+
+    attr_reader :package
+
+    def_delegators :@package, :dir, :package_name
+
+    def initialize(package)
+      @package = package
+    end
+
+    def self.create(package)
+      executable = new(package)
+      executable.copy_wrapper
+      executable
+    end
+
+    def copy_wrapper
+      print_to_console 'Creating exexutable...'
+
+      FileUtils.cp 'packaging/wrapper.sh', "#{dir}/#{package_name}"
+    end
+  end
+end

data/tasks/distribution/package.rb

@@ -0,0 +1,85 @@
+require 'fileutils'
+require 'forwardable'
+
+Dir[File.join(Dir.pwd, 'tasks', '**', '*.rb')].each { |f| require f }
+
+module Distribution
+  class Package
+    extend Forwardable
+    include PackageHelpers
+
+    attr_reader :config, :dir, :tarball, :package, :arch, :root
+
+    def_delegators :@config, :version, :rb_version, :package_name,
+                   :packaging_dir, :native_extensions
+
+    def initialize(arch)
+      abort 'Ruby 2.1.x required' if RUBY_VERSION !~ /^2\.1\./
+
+      @arch = arch
+      @config = ::Distribution.configuration
+      @dir = "#{package_name}-#{version}-#{arch}"
+      @package = self
+      @root = File.expand_path '.'
+    end
+
+    def self.create(args)
+      new(*args).build
+    end
+
+    def build
+      initialize_install_dir
+      copy_datapimp
+      install_ruby_and_gems
+      create_executable
+      post_cleanup
+      @tarball = create_tarball
+      clean_dir
+    end
+
+    private
+
+    def clean_dir
+      FileUtils.cd root do
+        FileUtils.remove_dir(dir, true) if Dir.exist? dir
+      end
+    end
+
+    def post_cleanup
+      print_to_console 'Cleaning up...'
+
+      files = ["#{packaging_dir}/traveling-ruby-#{rb_version}-#{arch}.tar.gz"]
+
+      files.each { |file| FileUtils.rm file if File.exist? file }
+    end
+
+    def create_tarball
+      Tarball.new self
+    end
+
+    def create_executable
+      Executable.create self
+    end
+
+    def install_ruby_and_gems
+      TravellingRuby.install self
+    end
+
+    def initialize_install_dir
+      clean_dir
+
+      FileUtils.cd root do
+        FileUtils.mkdir_p "#{dir}/lib/app"
+      end
+    end
+
+    def copy_datapimp
+      print_to_console 'Copying datapimp...'
+
+      %w(datapimp.gemspec Gemfile Gemfile.lock lib bin)
+        .each do |folder|
+        FileUtils.cp_r File.join(root, folder), "#{dir}/lib/app"
+      end
+    end
+  end
+end

data/tasks/distribution/package_helpers.rb

@@ -0,0 +1,12 @@
+module Distribution
+  module PackageHelpers
+    def curl(file)
+      system "curl -L -O --fail --silent #{file}"
+    end
+
+    def print_to_console(msg)
+      arch = package.arch
+      puts "[#{arch}]:" + ' ' * (16 - arch.size) + '=>' + ' ' + msg
+    end
+  end
+end

data/tasks/distribution/release.rb

@@ -0,0 +1,49 @@
+Dir[File.join(Dir.pwd, 'tasks', '**', '*.rb')].each { |f| require f }
+
+require 'digest'
+require 'octokit'
+require 'pathname'
+
+module Distribution
+  class Release
+    extend Forwardable
+    include PackageHelpers
+
+    attr_reader :tarball, :github, :package
+
+    def_delegators :@tarball, :package, :version, :file
+
+    def initialize(tarball)
+      @tarball = tarball
+      @github = Octokit::Client.new access_token: ENV['OCTODOWN_TOKEN']
+    end
+
+    def self.create(tarball)
+      release = new(tarball)
+      release.create_new_release
+    end
+
+    def create_new_release
+      print_to_console 'Publishing release to GitHub...'
+      github.create_release(
+        'datapimp/datapimp',
+        "v#{version}",
+        name: "v#{version}",
+        body: ReleaseNotes.new.content
+      )
+    end
+
+    def upload_asset
+      print_to_console 'Uploading to GitHub...'
+      github.upload_asset find_upload_url, file
+    end
+
+    private
+
+    def find_upload_url
+      Octokit.releases('datapimp/datapimp').find do |n|
+        n.tag_name == "v#{version}"
+      end[:url]
+    end
+  end
+end