datapimp 1.0.0 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 25225d44e31870eb3e377a8fa095cc4453e49648
-  data.tar.gz: 2d1998a952fd6d5da7f16c67bc5ceb8441637547
+  metadata.gz: a1b62d4316928f5d3ffa350d05cb579649fa1a67
+  data.tar.gz: d77cae1fa5c88ac3dedd9711dd8200cc98de1a6f
 SHA512:
-  metadata.gz: 50f0280170efb19fa6adaaf3b625e35ffb86cba675a5be09e8deee7c8f594ed988bd7649e142cb5c316905bf01d5afaa6f6de25d3e8450d758cead505bd48c47
-  data.tar.gz: e0f1f525c2b4a91902aef779128b67069be36d182685f0a2186446e471db9569370128a0003dc65c12a4e9f0cd7a8516b82c97286c09968594a9ac60e1fd1a95
+  metadata.gz: f373b48034f1ca5382f7e19f81cb86d623fa143ec22d72cc5f924115dae2b137d5d4103771fb471593bdc794a9498f34cba989e5d547a3903b87cfdd4ed24c08
+  data.tar.gz: 3e1df1aef41dfa912ebadde158c70fc8991869527d2b5b6f019739096110aa918f603e01c47e7029e6bdffdec97604b8fd022da7f3e2d5a4b78fbe1937651d6f

data/README.md

@@ -1,211 +1,39 @@
-### Data driven API programming  
+### Tools for working with common data sources
 
-This gem promotes a declarative style of developing JSON APIs that automates a lot of common patterns
-found in the development of systems which have multiple users, with multiple roles and permissions. 
+A collection of CLI utilities for pulling both files and structured data
+from file sharing and collaboration services like Google Drive and
+Dropbox.
 
-An API is a collection of resources, and resources can be queried, or commands can be run against resources, each by certain users in certain ways depending on the user's authorization level.  
+You can use this to pull down google spreadsheets and convert them to
+JSON structures, or to convert an excel spreadsheet on Dropbox to the
+same.
 
-The DSL provided by this gem allows you to accomplish the tasks of writing your API documentation, your integration tests, as well as your actual implementation code at the same time, and in such a way that exposes metadata. 
+You can use this to sync the contents of a local folder on your system
+with the remote contents of a file share on Dropbox or Google Drive.
 
-Because the API exposes metadata about the resources it contains, their schemas, and the various policies and details about them, it is possible for API client libraries to configure themselves accordingly.  
+And much more.
 
-### Example
+### Getting Started
 
-```ruby
-  require 'datapimp/dsl'
-
-  api :my_app => "My Application" do
-    version :v1
-
-    desc "Public users include anyone with access to the URL"
-    policy :public_users do
-      allow :books, :commands => false, :queries => true
-    end
-
-    desc "Authenticated users register and are given an auth token"
-    policy :logged_in_users do
-      authenticate_with :header => 'X-AUTH-TOKEN', :param => :auth_token
-      allow :books, :commands => true, :queries => true
-    end
-
-    desc "Admin users have the admin flag set to true"
-    policy :admin_users do
-      extends :logged_in_users
-      test :admin?
-    end
-  end
-```
-
-An API can be inspected:
-
-```ruby
-api("My Application").authentication_header #=> "X-AUTH-TOKEN"
-api("My Application").policies #=> [:public_users, :logged_in_users, :admin_users]
-api("My Application").policy(:admin_users).resource(:books).allowed_commands #=> [:create, :update, :delete]
-```
-
-An API is made up of resources:
-
-```ruby
-  resource "Books" do
-    serializer do
-      desc "A unique id for the book", :type => :integer
-      attribute :id
-
-      desc "The title of the book", :type => :string
-      attribute :title
-
-      desc "The year the book was published", :type => :integer
-      attribute :year
-
-      desc "A reference to the author", :type => "Author"
-      has_one :author
-    end
-
-    command :update, "Update a book's attributes" do
-      # Will ensure the command is run with
-      # Book.accessible_to(current_user).find(id).
-      scope :accessible_to
-
-      params do
-        duck :id, :method => :to_s
-
-        optional do
-          string :title
-        end
-      end
-    end
-
-    query do
-      start_from :scope => :accessible_to
-
-      params do
-        desc "The year the book was published (example: YYYY)"
-        integer :year_published
-      end
-
-      role :admin do
-        start_from :scope => :all
-      end
-    end
-  end
+```bash
+gem install datapimp
+datapimp help
 ```
 
-A resource can also be inspected:
-
-```ruby
-meta_data = api("My Application").resource("Books").meta_data
+#### Available Commands
 
-meta_data.attributes # {:id => "The id of the book", :year_published => "The year it was published"}
-meta_data.commands => [:update]
-meta_data.command(:update).arguments #=> [:id, :year, :title]
-meta_data.command(:update).optional_arguments #=> [:year, :title]
 ```
+  COMMANDS:
 
-This inspection goes a long way to some advanced features, such as
-automated documentation and integration test generation, or in writing
-tools for generating client libraries and the like.
+    config               Shows the configuration options being used
+    config set           manipulate configuration settings
+    help                 Display global or [command] help documentation
+    list spreadsheets    list the spreadsheets which can be used as datasources
+    setup amazon         setup integration with amazon
+    setup dropbox        setup integration with dropbox
+    setup github         setup integration with github
+    setup google         setup integration with google drive
+    sync data            Synchronize the contents of a local data store with its remote source
+    sync folder          Synchronize the contents of a local folder with a file sharing service
 
-### Customizing the elements
-
-How are each of these behaviors is stored in code? In a way that will be
-very familiar to Rails developers, following common naming conventions
-and file organization patterns.
-
-```
-- app
-  - commands
-    - application_command.rb
-    - create_book.rb
-    - update_book.rb
-  - contexts
-    - application_context.rb
-    - book_context.rb
-  - serializers
-    - book_serializer.rb
 ```
-
-### Request Context: Current User, Resource, and REST 
-
-From the programmer's perspective, a typical resource is made up of several request patterns:
-
-- Filter Context (index, show)
-- Commands (aka mutations. create, update, destroy)
-- Serializers (aka presenters, views) 
-
-Each of these objects can be configured to behave in certain ways that may be dependent on the user or role making the request to interact with them.
-
-Most API requests can be thought of in the following ways:
-
-```ruby
-# A Typical read request ( query / filter or detail view )
-
-response = present( this_resource )      # resource -> filter context
-            .to(this_user)               # filter context: relevant for this user
-            .in(this_presentation)       # serializer: different slices / renderings
-
-response.cache_key                       # russian doll style / max-updated-at friendly
-response.etag                            # http client conditional get
-```
-
-The filter context and serializer classes make this easy.  They also
-make writing -- or rather, generating -- documentation and tests very
-easy as well.
-
-```ruby
-# Typical mutation request ( create, update, delete )
-
-outcome = run(this_command)
-            .as(this_user)
-            .against(this_set_of_one_or_more_records)
-            .with(these_arguments)
-
-outcome.success? 
-
-outcome.error_messages
-
-outcome.result
-```
-
-The command class determines the specifics of the above style of
-request.
-
-### The Filter Context
-
-The filter context system is used to standardize the way we write
-typical index and show actions in a typical Rails app. A user is
-requesting to view a set of records, or an individual records.
-
-Given a user making a request to view a specific resource, we arrive at
-the 'filter context'.  The filter context is responsible for 'scoping' a
-resource to the set of records that user is permitted to view.  
-
-Based on the combination of parameters used to build that filter, we
-compute a cache key that simplifies the process of server caching and
-http client caching at the same time.
-
-The filter context itself and the available parameters and their allowed
-values are specified by the DSL, which simplifies the process of writing
-complex queries and also provides configuration meta-data that aids in
-the process of developing client user interfaces, API documentation, and
-test code.
-
-### Commands
-
-The command class allows you to declare the available parameters, the
-required values, their data types, etc.  It also allows you to declare
-which users can run the command, and further restrict the parameters
-allowed and the values they accept.
-
-### Serializers 
-
-- ActiveModel Serializers
-- Documentation DSL
-- Metadata for inspection + documentation generation
-
-## API Documentation & Integration Tests
-
-- rspec_api_documentation gem
-- plan: take advantage of metadata defined above to auto-generate
-  documentation with the ability to pass expectation blocks as pass /
-  fail indicators

data/datapimp.gemspec

@@ -18,11 +18,11 @@ Gem::Specification.new do |spec|
   spec.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   
   spec.add_dependency 'pry'
-  spec.add_dependency 'hashie', '>= 0.3.0'
+  spec.add_dependency 'hashie'
   spec.add_dependency 'colored'
   spec.add_dependency 'commander'
   spec.add_dependency 'fog-aws'
-  spec.add_dependency 'dropbox-api'
+  spec.add_dependency 'dropbox-api', '> 0.4.3'
   spec.add_dependency 'google_drive'
   spec.add_dependency 'rack-contrib'
   spec.add_dependency 'uri_template'

data/lib/datapimp/cli/config.rb

@@ -1,3 +1,20 @@
+command 'config' do |c|
+  c.syntax = "datapimp config [OPTIONS]"
+  c.description = "Shows the configuration options being used"
+
+  c.option '--env', "Output compatible with .env files"
+
+  c.action do |args, options|
+    Datapimp.config.current.to_hash.each do |key, value|
+      if options.env
+        puts "#{ key.to_s.upcase }= '#{ value }'"
+      else
+        puts "#{key}: #{value}"
+      end
+    end
+  end
+end
+
 command 'config set' do |c|
   c.syntax = 'datapimp config set KEY=VALUE KEY=VALUE [options]'
   c.description = 'manipulate configuration settings'

data/lib/datapimp/cli/list.rb

@@ -0,0 +1,15 @@
+command "list spreadsheets" do |c|
+  c.syntax = "datapimp list spreadsheets"
+  c.description = "list the spreadsheets which can be used as datasources"
+
+  c.option '--type TYPE', String, "What type of source data is this? #{ Datapimp::Sync.data_source_types.join(", ") }"
+
+  Datapimp::Cli.accepts_keys_for(c, :google, :dropbox)
+
+  c.action do |args, options|
+    Datapimp::Sync.google.spreadsheets.each do |sheet|
+      puts "#{ sheet.key }\t\t#{ sheet.title }"
+    end
+  end
+end
+

data/lib/datapimp/cli/support/data_sync.rb

@@ -0,0 +1,22 @@
+module Datapimp::DataSync
+  def self.sync_google_spreadsheet(options, *args)
+    require 'google_drive' unless defined?(::GoogleDrive)
+
+    raise 'Must setup google client' unless Datapimp::Sync.google.spreadsheets
+
+    key = args.shift
+    name = args.shift || "Spreadsheet"
+
+    raise 'Must supply a spreadsheet key' unless key
+
+    spreadsheet = Datapimp::Sources::GoogleSpreadsheet.new(name, key: key)
+
+    if options.output
+      Pathname(options.output).open("w+") do |fh|
+        fh.write(spreadsheet.to_s)
+      end
+    else
+      puts spreadsheet.to_s
+    end
+  end
+end

data/lib/datapimp/cli/sync.rb

@@ -3,11 +3,23 @@ command "sync folder" do |c|
   c.syntax = "datapimp sync folder LOCAL_PATH REMOTE_PATH [OPTIONS]"
 
   c.option '--type TYPE', String, "Which service is hosting the folder"
+  c.option '--action ACTION', String, "Which sync action to run? push, pull"
 
   Datapimp::Cli.accepts_keys_for(c, :amazon, :google, :github, :dropbox)
 
   c.action do |args, options|
+    options.default(action:"pull", type: "dropbox")
 
+    local, remote = args
+
+    folder = case
+      when options.type == "dropbox"
+        Datapimp::Sync::DropboxFolder.new(local: local, remote: remote)
+      when options.type == "google"
+        Datapimp::Sync::GoogleDriveFolder.new(local: local, remote: remote)
+    end
+
+    folder.run(options.action)
   end
 end
 
@@ -20,10 +32,13 @@ command "sync data" do |c|
   c.option '--columns NAMES', Array, "Extract only these columns"
 
   c.example "Syncing an excel file from dropbox ", "datapimp sync data --type dropbox --columns name,description --dropbox-app-key ABC --dropbox-app-secret DEF --dropbox-client-token HIJ --dropbox-client-secret JKL spreadsheets/test.xslx"
+  c.example "Syncing a google spreadsheet", "datapimp sync data --type google-spreadsheet WHATEVER_THE_KEY_IS"
 
   Datapimp::Cli.accepts_keys_for(c, :google, :github, :dropbox)
 
   c.action do |args, options|
-
+    if options.type == "google-spreadsheet"
+      Datapimp::DataSync.sync_google_spreadsheet(options, args)
+    end
   end
 end

data/lib/datapimp/clients/dropbox.rb

@@ -72,27 +72,6 @@ module Datapimp
         end
       end
 
-      def sync(local_path, remote_path, options={})
-        sync_folders(local_path, remote_path, options)
-      end
-
-      # Instructs Datapimp to create a syncable folder
-      # between the local and remote path, and optionally
-      # modifies the middleman config for the site
-      def sync_folders(local_path, remote_path, options={})
-        app = options[:app]
-
-        folder = Datapimp::Sync::Folder.new(local_path: local_path,
-                                            remote_path: remote_path,
-                                            app: app).synced
-
-        if !!options[:append_config] == true
-          Datapimp.append_config(folder.config_line)
-        end
-
-        folder
-      end
-
       def method_missing meth, *args, &block
         if api.respond_to?(meth)
           return api.send(meth, *args, &block)

data/lib/datapimp/sources/google.rb

@@ -0,0 +1,212 @@
+module Datapimp::Sources
+  class GoogleSpreadsheet < Datapimp::Sources::Base
+    requires :key
+
+    attr_accessor :key,
+                  :session,
+                  :name
+
+    def initialize name, options={}
+      @options  = options
+
+      if name.is_a?(GoogleDrive::Spreadsheet)
+        @spreadsheet = name
+        @name = @spreadsheet.title
+        @key = @spreadsheet.key
+      end
+
+      @key      ||= options[:key]
+      @session  ||= options.fetch(:session) { Datapimp::Sync.google.api }
+
+      ensure_valid_options!
+    end
+
+    def self.create_from_file(path, title)
+      if find_by_title(title)
+        raise 'Spreadsheet with this title already exists'
+      end
+
+      session.upload_from_file(path, title, :content_type => "text/csv")
+
+      find_by_title(title)
+    end
+
+    def self.[](key_or_title)
+      find_by_key(key_or_title) || find_by_title(key_or_title)
+    end
+
+    def self.find_by_key(key)
+      sheet = session_spreadsheets.detect do |spreadsheet|
+        spreadsheet.key == key
+      end
+
+      sheet && new(sheet, session: Datapimp::Sync.google.session)
+    end
+
+    def self.find_by_title title
+      sheet = session_spreadsheets.detect do |spreadsheet|
+        spreadsheet.title.match(title)
+      end
+
+      sheet && new(sheet, session: Datapimp::Sync.google.session)
+    end
+
+    def self.session_spreadsheets
+      @session_spreadsheets ||= Datapimp::Sync.google.api.spreadsheets
+    end
+
+    def self.create_from_data(data, options={})
+      require 'csv'
+
+      headers = Array(options[:headers]).map(&:to_s)
+
+      tmpfile = "tmp-csv.csv"
+
+      CSV.open(tmpfile, "wb") do |csv|
+        csv << headers
+
+        data.each do |row|
+          csv << headers.map do |header|
+            row = row.stringify_keys
+            row[header.to_s]
+          end
+        end
+      end
+
+      spreadsheet = Datapimp::Sync.google.api.upload_from_file(tmpfile, options[:title], :content_type => "text/csv")
+
+      new(spreadsheet.title, key: spreadsheet.key)
+    end
+
+
+    def title
+      @name ||= spreadsheet.try(:title)
+    end
+
+    def edit_url
+      spreadsheet.human_url
+    end
+
+    def share_write_access_with *emails
+      acl = spreadsheet.acl
+
+      Array(emails).flatten.each do |email|
+        acl.push scope_type: "user",
+                 with_key: false,
+                 role: "writer",
+                 scope: email
+      end
+    end
+
+    def share_read_access_with *emails
+      acl = spreadsheet.acl
+
+      Array(emails).flatten.each do |email|
+        acl.push scope_type: "user",
+                 with_key: false,
+                 role: "reader",
+                 scope: email
+      end
+    end
+
+    def add_to_collection collection_title
+      collection = if collection_title.is_a?(GoogleDrive::Collection)
+        collection_title
+      else
+        session.collections.find do |c|
+          c.title == collection_title
+        end
+      end
+
+      if !collection
+        collection_names = session.collections.map(&:title)
+        raise 'Could not find collection in Google drive. Maybe you mean: ' + collection_names.join(', ')
+      end
+    end
+
+    def spreadsheet_key
+      key
+    end
+
+    def stale?
+      (!need_to_refresh? && (age > max_age)) || fresh_on_server?
+    end
+
+    def fresh_on_server?
+      refreshed_at.to_i > 0 && (last_updated_at > refreshed_at)
+    end
+
+    def last_updated_at
+      if value = spreadsheet.document_feed_entry_internal.css('updated').try(:text) rescue nil
+        DateTime.parse(value).to_i
+      else
+        Time.now.to_i
+      end
+    end
+
+    def fetch
+      self.raw = process_worksheets
+    end
+
+    def preprocess
+      single? ? raw.values.flatten : raw
+    end
+
+    protected
+
+      def process_worksheets
+        worksheets.inject({}.to_mash) do |memo, parts|
+          k, ws = parts
+          header_row = Array(ws.rows[0])
+          column_names = header_row.map {|cell| "#{ cell }".parameterize.underscore }
+          rows = ws.rows.slice(1, ws.rows.length)
+
+          row_index = 1
+          memo[k] = rows.map do |row|
+            col_index = 0
+
+            _record = column_names.inject({}) do |record, field|
+              record[field] = "#{ row[col_index] }".strip
+              record["_id"] = row_index
+              col_index += 1
+              record
+            end
+
+            row_index += 1
+
+            _record
+          end
+
+          memo
+        end
+      end
+
+      def single?
+        worksheets.length == 1
+      end
+
+      def header_rows_for_worksheet key
+        if key.is_a?(Fixnum)
+          _worksheets[key]
+        else
+          worksheets.fetch(key)
+        end
+      end
+
+      def worksheets
+        @worksheets ||= _worksheets.inject({}.to_mash) do |memo,ws|
+          key = ws.title.strip.downcase.underscore.gsub(/\s+/,'_')
+          memo[key] = ws
+          memo
+        end
+      end
+
+      def _worksheets
+        @_worksheets ||= spreadsheet.worksheets
+      end
+
+      def spreadsheet
+        @spreadsheet ||= session.spreadsheet_by_key(spreadsheet_key)
+      end
+  end
+end

data/lib/datapimp/sources.rb

@@ -0,0 +1,198 @@
+# The `Datapimp::Sources` module houses the various
+# types of remote data stores we are reading and converting into
+# a JSON array of objects that gets cached on our filesystem.
+#
+module Datapimp
+  module Sources
+    class Base
+      attr_reader :options, :name
+      attr_accessor :raw, :processed, :format, :scopes, :slug_column, :refreshed_at, :path
+
+      class << self
+        attr_accessor :required_options
+      end
+
+      def self.requires *args
+        self.required_options = args
+      end
+
+      def initialize(name, options={})
+        @name     ||= name
+        @options  ||= options
+        @format   ||= options.fetch(:format, :json)
+        @path     ||= options.fetch(:path) { Pathname(Dir.pwd()) }
+
+        @slug_column = options.fetch(:slug_column, :_id)
+
+        ensure_valid_options!
+      end
+
+      def to_s
+        data.to_json
+      end
+
+      # defines a scope for the records in this data source
+      # a scope is a named filter, implemented in the form of a block
+      # which is passed each record.  if the block returns true, it returns
+      # the record:
+      #
+      # Example:
+      #
+      # data_source(:galleries) do
+      #   scope :active, -> {|record| record.state == "active" }
+      # end
+      def scope(*args, block)
+        name = args.first
+        (self.scopes ||= {})[name.to_sym] = block
+      end
+
+      def has_scope?(scope_name)
+        scope_name && (self.scopes ||= {}).key?(scope_name.to_sym)
+      end
+
+      # compute properties takes the raw data of each record
+      # and sets additional properties on the records which may
+      # not be persited in the data source
+      def compute_properties
+        self.processed && self.processed.map! do |row|
+          if slug_column && row.respond_to?(slug_column)
+            row.slug = row.send(slug_column).to_s.parameterize
+          end
+
+          row
+        end
+
+        processors.each do |processor|
+          original = self.processed.dup
+          modified = []
+
+          original.each_with_index do |record, index|
+            previous = original[index - 1]
+            modified.push(processor.call(record, index, previous: previous, set: original))
+          end
+
+          self.processed = modified
+        end
+      end
+
+      def processors &block
+        @processors ||= []
+        @processors << block if block_given?
+        @processors
+      end
+
+      # makes sure that the required options for this data source
+      # are passed for any instance of the data source
+      def ensure_valid_options!
+        missing_options = (Array(self.class.required_options) - options.keys.map(&:to_sym))
+
+        missing_options.reject! do |key|
+          respond_to?(key) && !send(key).nil?
+        end
+
+        if missing_options.length > 0
+          raise 'Error: failure to supply the following options: ' + missing_options.map(&:to_s).join(",")
+        end
+      end
+
+      def select(&block)
+        data.select(&block)
+      end
+
+      def refresh
+        fetch
+        process
+        self.refreshed_at = Time.now.to_i
+        self
+      end
+
+      def refresh_if_stale?
+        refresh! if stale?
+      end
+
+      # A data source is stale if it has been populated
+      # and the age is greater than the max age we allow.
+      def stale?
+        !need_to_refresh? && (age > max_age)
+      end
+
+      def fresh_on_server?
+        need_to_refresh?
+      end
+
+      def max_age
+        max = ENV['MAX_DATA_SOURCE_AGE']
+        (max && max.to_i) || 120
+      end
+
+      # how long since this data source has been refreshed?
+      def age
+        Time.now.to_i - refreshed_at.to_i
+      end
+
+      def data
+        refresh if need_to_refresh?
+        processed
+      end
+
+      def refresh!
+        refresh
+        save_to_disk
+      end
+
+      def need_to_refresh?
+        !(@fetched && @_processed)
+      end
+
+      def fetch
+        @fetched = true
+        self.raw = []
+      end
+
+      def preprocess
+        self.raw.dup
+      end
+
+      def process
+        @_processed = true
+        self.processed = preprocess
+        # set_id
+        compute_properties
+        self.processed
+      end
+
+      def refreshed_at
+        return @refreshed_at if @refreshed_at.to_i > 0
+
+        if path_to_file.exist?
+          @refreshed_at = File.mtime(path.join(file)).to_i
+        end
+      end
+
+      def save_to_disk
+        unless path_to_file.dirname.exist?
+          FileUtils.mkdir(path_to_file.dirname)
+        end
+
+        path_to_file.open('w+') {|fh| fh.write(to_s) }
+      end
+
+      def persisted?
+        path_to_file && path_to_file.exist?
+      end
+
+      def file
+        @file ||= name.parameterize if name.respond_to?(:parameterize)
+        @file.gsub!("-","_")
+        @file = "#{@file}.json" unless @file.match(/\.json/i)
+        @file
+      end
+
+      def path_to_file
+        Pathname(path).join("#{ file }")
+      end
+    end
+  end
+end
+
+Dir[Datapimp.lib.join("datapimp/sources/**/*.rb")].each {|f| require(f) }

data/lib/datapimp/sync/dropbox_folder.rb

@@ -1,10 +1,37 @@
 module Datapimp
-  class Sync::DropboxFolder < OpenStruct
-    class_attribute :default_path_prefix, :default_root
+  class Sync::DropboxFolder < Hashie::Mash
+    def dropbox
+      @dropbox ||= Datapimp::Sync.dropbox
+    end
+
+    def delta
+      @delta ||= dropbox.delta(cursor, remote_path)
+    end
+
+    def local_path
+      Pathname(local)
+    end
+
+    def remote_path
+      Datapimp::Sync.dropbox.ls(remote)
+    end
+
+    def cursor
+      cursor_path.exist? && cursor_path.read
+    end
+
+    def cursor_path
+      local_path.join('.dropbox-cursor')
+    end
+
+    def run(action)
+      action = action.to_sym
+
+      if action == :push
 
+      elsif action == :pull
 
-    def push
-      Datapimp.dropbox(token: client_token, secret: client_secret)
+      end
     end
   end
 end

data/lib/datapimp/version.rb

@@ -1,3 +1,3 @@
 module Datapimp
-  VERSION = "1.0.0"
+  VERSION = "1.0.1"
 end

data/lib/datapimp.rb

@@ -3,6 +3,7 @@ require 'set'
 require 'pathname'
 require 'hashie'
 require 'datapimp/core_ext'
+require 'active_support'
 
 module Datapimp
   def self.config
@@ -29,5 +30,7 @@ end
 
 require 'datapimp/version'
 require 'datapimp/configuration'
-require 'datapimp/data_sources'
+require 'datapimp/sources'
 require 'datapimp/sync'
+require 'datapimp/sync/dropbox_folder'
+require 'datapimp/sync/google_drive_folder'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: datapimp
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.0.1
 platform: ruby
 authors:
 - Jonathan Soeder
@@ -30,14 +30,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.3.0
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.3.0
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: colored
   requirement: !ruby/object:Gem::Requirement
@@ -84,16 +84,16 @@ dependencies:
   name: dropbox-api
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - ">"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.4.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - ">"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.4.3
 - !ruby/object:Gem::Dependency
   name: google_drive
   requirement: !ruby/object:Gem::Requirement
@@ -296,7 +296,9 @@ files:
 - lib/datapimp/cli.rb
 - lib/datapimp/cli/01_extensions.rb
 - lib/datapimp/cli/config.rb
+- lib/datapimp/cli/list.rb
 - lib/datapimp/cli/setup.rb
+- lib/datapimp/cli/support/data_sync.rb
 - lib/datapimp/cli/sync.rb
 - lib/datapimp/clients/amazon.rb
 - lib/datapimp/clients/dropbox.rb
@@ -304,13 +306,13 @@ files:
 - lib/datapimp/clients/google.rb
 - lib/datapimp/configuration.rb
 - lib/datapimp/core_ext.rb
-- lib/datapimp/data_sources.rb
-- lib/datapimp/data_sources/dropbox.rb
-- lib/datapimp/data_sources/excel.rb
-- lib/datapimp/data_sources/github.rb
-- lib/datapimp/data_sources/google.rb
-- lib/datapimp/data_sources/json.rb
-- lib/datapimp/data_sources/nokogiri.rb
+- lib/datapimp/sources.rb
+- lib/datapimp/sources/dropbox.rb
+- lib/datapimp/sources/excel.rb
+- lib/datapimp/sources/github.rb
+- lib/datapimp/sources/google.rb
+- lib/datapimp/sources/json.rb
+- lib/datapimp/sources/nokogiri.rb
 - lib/datapimp/sync.rb
 - lib/datapimp/sync/dropbox_delta.rb
 - lib/datapimp/sync/dropbox_folder.rb

data/lib/datapimp/data_sources/google.rb

@@ -1,5 +0,0 @@
-module Datapimp::Sources
-  class Google < OpenStruct
-
-  end
-end

data/lib/datapimp/data_sources.rb

@@ -1,10 +0,0 @@
-# The `Datapimp::Dataources` module houses the various
-# types of remote data stores we are reading and converting into
-# a JSON array of objects that gets cached on our filesystem.
-module Datapimp
-  module DataSources
-
-  end
-end
-
-Dir[Datapimp.lib.join("datapimp/data_sources/**/*.rb")].each {|f| require(f) }