datapimp 0.0.1 → 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 25225d44e31870eb3e377a8fa095cc4453e49648
+  data.tar.gz: 2d1998a952fd6d5da7f16c67bc5ceb8441637547
+SHA512:
+  metadata.gz: 50f0280170efb19fa6adaaf3b625e35ffb86cba675a5be09e8deee7c8f594ed988bd7649e142cb5c316905bf01d5afaa6f6de25d3e8450d758cead505bd48c47
+  data.tar.gz: e0f1f525c2b4a91902aef779128b67069be36d182685f0a2186446e471db9569370128a0003dc65c12a4e9f0cd7a8516b82c97286c09968594a9ac60e1fd1a95

data/.gitignore

@@ -15,3 +15,5 @@ spec/reports
 test/tmp
 test/version_tmp
 tmp
+*.log
+dev-branch/

data/Gemfile

@@ -2,3 +2,9 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in datapimp.gemspec
 gemspec
+
+group :development do
+  gem 'pry'
+  gem 'pry-nav'
+  gem 'active_model_serializers'
+end

data/Guardfile

@@ -0,0 +1,5 @@
+guard 'rspec', :all_on_start => true do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/(.+)\.rb$})     { |m| "spec/lib/#{m[1]}_spec.rb" }
+  watch('spec/spec_helper.rb')  { "spec/" }
+end

data/LICENSE.txt

@@ -1,22 +1,8 @@
 Copyright (c) 2013 Jonathan Soeder
 
-MIT License
+Permission to use this gem is granted on a project by project basis. 
 
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
+For open source projects, the license is identical to the MIT license.
 
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+For commercial ventures, the license is subject to the completion of a
+contractual agreement between the company and Jonathan Soeder. 

data/README.md

@@ -1,29 +1,211 @@
-# Datapimp
+### Data driven API programming  
 
-TODO: Write a gem description
+This gem promotes a declarative style of developing JSON APIs that automates a lot of common patterns
+found in the development of systems which have multiple users, with multiple roles and permissions. 
 
-## Installation
+An API is a collection of resources, and resources can be queried, or commands can be run against resources, each by certain users in certain ways depending on the user's authorization level.  
 
-Add this line to your application's Gemfile:
+The DSL provided by this gem allows you to accomplish the tasks of writing your API documentation, your integration tests, as well as your actual implementation code at the same time, and in such a way that exposes metadata. 
 
-    gem 'datapimp'
+Because the API exposes metadata about the resources it contains, their schemas, and the various policies and details about them, it is possible for API client libraries to configure themselves accordingly.  
 
-And then execute:
+### Example
 
-    $ bundle
+```ruby
+  require 'datapimp/dsl'
 
-Or install it yourself as:
+  api :my_app => "My Application" do
+    version :v1
 
-    $ gem install datapimp
+    desc "Public users include anyone with access to the URL"
+    policy :public_users do
+      allow :books, :commands => false, :queries => true
+    end
 
-## Usage
+    desc "Authenticated users register and are given an auth token"
+    policy :logged_in_users do
+      authenticate_with :header => 'X-AUTH-TOKEN', :param => :auth_token
+      allow :books, :commands => true, :queries => true
+    end
 
-TODO: Write usage instructions here
+    desc "Admin users have the admin flag set to true"
+    policy :admin_users do
+      extends :logged_in_users
+      test :admin?
+    end
+  end
+```
 
-## Contributing
+An API can be inspected:
 
-1. Fork it
-2. Create your feature branch (`git checkout -b my-new-feature`)
-3. Commit your changes (`git commit -am 'Add some feature'`)
-4. Push to the branch (`git push origin my-new-feature`)
-5. Create new Pull Request
+```ruby
+api("My Application").authentication_header #=> "X-AUTH-TOKEN"
+api("My Application").policies #=> [:public_users, :logged_in_users, :admin_users]
+api("My Application").policy(:admin_users).resource(:books).allowed_commands #=> [:create, :update, :delete]
+```
+
+An API is made up of resources:
+
+```ruby
+  resource "Books" do
+    serializer do
+      desc "A unique id for the book", :type => :integer
+      attribute :id
+
+      desc "The title of the book", :type => :string
+      attribute :title
+
+      desc "The year the book was published", :type => :integer
+      attribute :year
+
+      desc "A reference to the author", :type => "Author"
+      has_one :author
+    end
+
+    command :update, "Update a book's attributes" do
+      # Will ensure the command is run with
+      # Book.accessible_to(current_user).find(id).
+      scope :accessible_to
+
+      params do
+        duck :id, :method => :to_s
+
+        optional do
+          string :title
+        end
+      end
+    end
+
+    query do
+      start_from :scope => :accessible_to
+
+      params do
+        desc "The year the book was published (example: YYYY)"
+        integer :year_published
+      end
+
+      role :admin do
+        start_from :scope => :all
+      end
+    end
+  end
+```
+
+A resource can also be inspected:
+
+```ruby
+meta_data = api("My Application").resource("Books").meta_data
+
+meta_data.attributes # {:id => "The id of the book", :year_published => "The year it was published"}
+meta_data.commands => [:update]
+meta_data.command(:update).arguments #=> [:id, :year, :title]
+meta_data.command(:update).optional_arguments #=> [:year, :title]
+```
+
+This inspection goes a long way to some advanced features, such as
+automated documentation and integration test generation, or in writing
+tools for generating client libraries and the like.
+
+### Customizing the elements
+
+How are each of these behaviors is stored in code? In a way that will be
+very familiar to Rails developers, following common naming conventions
+and file organization patterns.
+
+```
+- app
+  - commands
+    - application_command.rb
+    - create_book.rb
+    - update_book.rb
+  - contexts
+    - application_context.rb
+    - book_context.rb
+  - serializers
+    - book_serializer.rb
+```
+
+### Request Context: Current User, Resource, and REST 
+
+From the programmer's perspective, a typical resource is made up of several request patterns:
+
+- Filter Context (index, show)
+- Commands (aka mutations. create, update, destroy)
+- Serializers (aka presenters, views) 
+
+Each of these objects can be configured to behave in certain ways that may be dependent on the user or role making the request to interact with them.
+
+Most API requests can be thought of in the following ways:
+
+```ruby
+# A Typical read request ( query / filter or detail view )
+
+response = present( this_resource )      # resource -> filter context
+            .to(this_user)               # filter context: relevant for this user
+            .in(this_presentation)       # serializer: different slices / renderings
+
+response.cache_key                       # russian doll style / max-updated-at friendly
+response.etag                            # http client conditional get
+```
+
+The filter context and serializer classes make this easy.  They also
+make writing -- or rather, generating -- documentation and tests very
+easy as well.
+
+```ruby
+# Typical mutation request ( create, update, delete )
+
+outcome = run(this_command)
+            .as(this_user)
+            .against(this_set_of_one_or_more_records)
+            .with(these_arguments)
+
+outcome.success? 
+
+outcome.error_messages
+
+outcome.result
+```
+
+The command class determines the specifics of the above style of
+request.
+
+### The Filter Context
+
+The filter context system is used to standardize the way we write
+typical index and show actions in a typical Rails app. A user is
+requesting to view a set of records, or an individual records.
+
+Given a user making a request to view a specific resource, we arrive at
+the 'filter context'.  The filter context is responsible for 'scoping' a
+resource to the set of records that user is permitted to view.  
+
+Based on the combination of parameters used to build that filter, we
+compute a cache key that simplifies the process of server caching and
+http client caching at the same time.
+
+The filter context itself and the available parameters and their allowed
+values are specified by the DSL, which simplifies the process of writing
+complex queries and also provides configuration meta-data that aids in
+the process of developing client user interfaces, API documentation, and
+test code.
+
+### Commands
+
+The command class allows you to declare the available parameters, the
+required values, their data types, etc.  It also allows you to declare
+which users can run the command, and further restrict the parameters
+allowed and the values they accept.
+
+### Serializers 
+
+- ActiveModel Serializers
+- Documentation DSL
+- Metadata for inspection + documentation generation
+
+## API Documentation & Integration Tests
+
+- rspec_api_documentation gem
+- plan: take advantage of metadata defined above to auto-generate
+  documentation with the ability to pass expectation blocks as pass /
+  fail indicators

data/Rakefile

@@ -1 +1,25 @@
+Dir[File.join(Dir.pwd, 'tasks', '**', '*.rb')].each { |f| require f }
+Dir[File.join(Dir.pwd, 'tasks', '*.rake')].each { |f| load f }
+
 require "bundler/gem_tasks"
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+Distribution.configure do |config|
+  config.package_name = 'datapimp'
+  config.version = Datapimp::VERSION
+  config.rb_version = '20150210-2.1.5'
+  config.packaging_dir = File.expand_path 'packaging'
+  config.native_extensions = [
+    #'github-markdown-0.6.8',
+    #'escape_utils-1.0.1',
+    #'charlock_holmes-0.7.3',
+    #'posix-spawn-0.3.9',
+    #'nokogumbo-1.3.0',
+    #'rugged-0.21.4',
+    #'nokogiri-1.6.5',
+  ]
+end
+
+task :default => :spec

data/bin/datapimp

@@ -0,0 +1,18 @@
+#!/usr/bin/env ruby
+
+require "rubygems"
+require "pathname"
+
+$:.unshift Pathname(File.dirname(__FILE__)).join("..","lib")
+
+require "pry"
+require 'colored'
+require "commander/import"
+require 'datapimp'
+require 'datapimp/cli'
+
+program :name, 'Datapimp'
+program :version, Datapimp::VERSION
+program :description, 'Datapimp style command line utilities'
+
+Datapimp::Cli.load_commands

data/datapimp.gemspec

@@ -8,16 +8,36 @@ Gem::Specification.new do |spec|
   spec.version       = Datapimp::VERSION
   spec.authors       = ["Jonathan Soeder"]
   spec.email         = ["jonathan.soeder@gmail.com"]
-  spec.description   = %q{Write a gem description}
-  spec.summary       = %q{Write a gem summary}
+  spec.description   = %q{Your rails app in a custom tailored suit.}
+  spec.summary       = %q{A collection of API development patterns that I have accumulated in my career as a boss.}
   spec.homepage      = ""
   spec.license       = "MIT"
 
-  spec.files         = `git ls-files`.split($/)
-  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
-  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.files         = `git ls-files`.split("\n")
+  spec.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  spec.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  
+  spec.add_dependency 'pry'
+  spec.add_dependency 'hashie', '>= 0.3.0'
+  spec.add_dependency 'colored'
+  spec.add_dependency 'commander'
+  spec.add_dependency 'fog-aws'
+  spec.add_dependency 'dropbox-api'
+  spec.add_dependency 'google_drive'
+  spec.add_dependency 'rack-contrib'
+  spec.add_dependency 'uri_template'
+  spec.add_dependency 'dnsimple-ruby'
+  spec.add_dependency 'rack-proxy'
+  spec.add_dependency 'axlsx'
+  spec.add_dependency 'launchy'
+  spec.add_dependency 'oauth', '~> 0.4.7'
+  spec.add_dependency 'octokit', '>= 3.0.0'
+  
+  spec.add_development_dependency "rake", '~> 0'
+  spec.add_development_dependency "rack-test", '~> 0'
+  spec.add_development_dependency 'rspec', '~> 3.0'
+  spec.add_development_dependency "pry-nav", '~> 0'
+
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 1.3"
-  spec.add_development_dependency "rake"
 end

data/lib/datapimp/cli/01_extensions.rb

@@ -0,0 +1,25 @@
+class Commander::Command
+  def action(*args, &block)
+
+    wrapper = lambda do |a, options|
+      if options.config
+        read = Pathname(options.config).read
+        json = JSON.parse(read)
+
+        Datapimp.config.apply_all(json)
+      end
+
+      Datapimp.config.apply_all(options.to_hash)
+
+      block.call(a, options)
+    end
+
+    send(:when_called, *args, &wrapper)
+  end
+
+  class Options
+    def to_hash
+      __hash__
+    end
+  end
+end

data/lib/datapimp/cli/config.rb

@@ -0,0 +1,24 @@
+command 'config set' do |c|
+  c.syntax = 'datapimp config set KEY=VALUE KEY=VALUE [options]'
+  c.description = 'manipulate configuration settings'
+
+  c.option '--global', 'Set the configuration globally'
+  c.option '--local', 'Set the configuration globally'
+
+  c.example "set a bunch of config parameters", "datapimp config set DROPBOX_APP_KEY=xxx DROPBOX_APP_SECRET=yyy GITHUB_APP_SECRET=zzz"
+
+  c.action do |args, _options|
+    Datapimp::Configuration.initialize!
+
+    args.select { |pair| pair.match(/=/) }
+      .map { |pair| pair.split('=') }
+      .each do |group|
+        key, value = group
+        Datapimp.config.set(key, value, false, global: !!(_options.global))
+      end
+
+    Datapimp.config.save!
+
+    Datapimp.config.show
+  end
+end

data/lib/datapimp/cli/setup.rb

@@ -0,0 +1,35 @@
+command "setup google" do |c|
+  c.syntax = "datapimp set up google"
+  c.description = "setup integration with google drive"
+
+  c.action do |args, options|
+    Datapimp::Sync.google.setup()
+  end
+end
+
+command "setup amazon" do |c|
+  c.syntax = "datapimp setup amazon"
+  c.description = "setup integration with amazon"
+
+  c.action do |args, options|
+    Datapimp::Sync.amazon.interactive_setup()
+  end
+end
+
+command "setup dropbox" do |c|
+  c.syntax = "datapimp set up dropbox"
+  c.description = "setup integration with dropbox"
+
+  c.action do |args, options|
+    Datapimp::Sync.dropbox.setup()
+  end
+end
+
+command "setup github" do |c|
+  c.syntax = "datapimp set up github"
+  c.description = "setup integration with github"
+
+  c.action do |args, options|
+    Datapimp::Sync.github.setup()
+  end
+end

data/lib/datapimp/cli/sync.rb

@@ -0,0 +1,29 @@
+command "sync folder" do |c|
+  c.description = "Synchronize the contents of a local folder with a file sharing service"
+  c.syntax = "datapimp sync folder LOCAL_PATH REMOTE_PATH [OPTIONS]"
+
+  c.option '--type TYPE', String, "Which service is hosting the folder"
+
+  Datapimp::Cli.accepts_keys_for(c, :amazon, :google, :github, :dropbox)
+
+  c.action do |args, options|
+
+  end
+end
+
+command "sync data" do |c|
+  c.description = "Synchronize the contents of a local data store with its remote source"
+  c.syntax = "datapimp sync data [OPTIONS]"
+
+  c.option '--type TYPE', String, "What type of source data is this? #{ Datapimp::Sync.data_source_types.join(", ") }"
+  c.option '--output FILE', String, "Write the output to a file"
+  c.option '--columns NAMES', Array, "Extract only these columns"
+
+  c.example "Syncing an excel file from dropbox ", "datapimp sync data --type dropbox --columns name,description --dropbox-app-key ABC --dropbox-app-secret DEF --dropbox-client-token HIJ --dropbox-client-secret JKL spreadsheets/test.xslx"
+
+  Datapimp::Cli.accepts_keys_for(c, :google, :github, :dropbox)
+
+  c.action do |args, options|
+
+  end
+end

data/lib/datapimp/cli.rb

@@ -0,0 +1,49 @@
+require 'launchy'
+
+module Datapimp
+  module Cli
+    def self.load_commands(from_path=nil)
+      from_path ||= Datapimp.lib.join("datapimp","cli")
+      Dir[from_path.join("**/*.rb")].each {|f| require(f) }
+    end
+
+    def self.accepts_keys_for(c, *services)
+      services.map!(&:to_sym)
+
+      if services.include?(:dropbox)
+        c.option "--dropbox-app-key KEY", String, "The dropbox app key"
+        c.option "--dropbox-app-secret SECRET", String, "The dropbox app secret"
+        c.option "--dropbox-app-sandbox", "The dropbox app is a sandbox app"
+        c.option "--dropbox-client-token TOKEN", String, "The dropbox client token"
+        c.option "--dropbox-client-secret SECRET", String, "The dropbox client secret"
+        c.option "--dropbox-path-prefix PATH", String, "The path prefix for this folder (for sandboxed apps)"
+      end
+
+      if services.include?(:google)
+        c.option "--google-client-id KEY", String, "The google client id"
+        c.option "--google-client-secret KEY", String, "The google client secret"
+        c.option "--google-refresh-token", String, "The google refresh token"
+        c.option "--google-access-token KEY", String, "The google access token"
+      end
+
+      if services.include?(:dnsimple)
+        c.option "--dnsimple-api-token", String, "The DNSimple API Token"
+        c.option "--dnsimple-username", String, "The DNSimple Username"
+      end
+
+      if services.include?(:github)
+        c.option "--github-username", String, "The Github Username"
+        c.option "--github-organization", String, "The Github Organization"
+        c.option "--github-access-token", String, "The Github Personal Access Token"
+        c.option "--github-app-key", String, "The Github App Key"
+        c.option "--github-app-secret", String, "The Github App Secret"
+      end
+
+      if services.include?(:amazon) || services.include?(:aws)
+        c.option '--aws-secret-access-key', String, 'AWS Secret Access Key'
+        c.option '--aws-access-key-id', String, 'AWS Access Key ID'
+      end
+
+    end
+  end
+end