shydra 0.0.1

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,8 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in shydra.gemspec
+gemspec
+
+group :test do
+  gem "guard-rspec"
+end

data/Guardfile

@@ -0,0 +1,8 @@
+# More info at https://github.com/guard/guard#readme
+
+guard :rspec, cli: "--color", bundler: false, all_on_start: true do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/(.+)\.rb$})     { |m| "spec/#{m[1]}_spec.rb" }
+  watch('spec/spec_helper.rb')  { "spec" }
+end
+

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Michael Johnston
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,29 @@
+# Shydra
+
+A fast, parallel shopify api client using typhoeus/hydra. Attempts to balance being a good citizen about api limits and fetching multiple apai records in parallel.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'shydra'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install shydra
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,7 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new
+
+task :default => :spec
+task :test => :spec

data/lib/shydra.rb

@@ -0,0 +1,13 @@
+require "shydra/version"
+require "typhoeus"
+require "shopify_api"
+require 'shydra/request'
+require 'shydra/response'
+
+module Shydra
+  class << self
+    attr_accessor :max_concurrency
+  end
+end
+
+Shydra.max_concurrency = 20

data/lib/shydra/hydra.rb

@@ -0,0 +1,62 @@
+=begin
+
+  Scenarios:
+
+  1. request that might return more than 250
+    - limit: :none
+    - adds callback that fetches another page if result size == SHOPIFY_API_MAX_LIMIT
+
+  2. request that will return several pages that needs to be fast
+  - use a hydra
+  - does count first
+  - queues request for each page, callback inserts into correct slot in result_array
+  - if too many pages for api limit, fetch as many pages as possible, then sleep for some amount of time
+  - time to sleep: requests-used-before-starting / API-LIMIT
+  - at the end flatten the result_array 
+
+
+  API LIMIT
+  - memcached
+  - cache {used: x, at: Time.now}
+  - before starting hydra:
+    - count requests
+    - check cache
+    - if at < TUNEABLE_PARAM (like, 20 seconds) & enough requests (including TUNEABLE_BUFFER)
+      - run everything
+    - do_run: run 1 request. update cache
+      - if enough requests, run everything
+      - else bail or sleep(time allowed)
+      - after sleeping time allowed, goto: do_run
+    - because requests may have callbacks that queue other requests, the run_loop has to go through this alogrithm every time it pulls a new batch of requests off
+=end
+module Shydra
+  class Hydra < Typhoeus::Hydra
+
+
+    def initialize(options = {})
+      options[:max_concurrency] ||= Shydra.max_concurrency
+      super(options)
+    end
+
+
+
+    def pause!
+      @paused = true
+    end
+
+    def paused?
+      @paused
+    end
+
+    def run
+      @paused = false
+      super
+    end
+
+    def dequeue
+      return if paused?
+      super
+    end
+
+  end
+end

data/lib/shydra/request.rb

@@ -0,0 +1,64 @@
+require_relative 'response'
+
+module Shydra  
+  class Request < Typhoeus::Request
+    SHOPIFY_API_MAX_LIMIT = 250
+
+    attr_accessor :resource, :data_root
+    attr_accessor :options
+
+    def self.base_uri
+      ShopifyAPI::Base.site
+    end
+
+    def initialize(*args)
+      @options = args.extract_options!
+      @count = @options.delete(:count)
+      if args[1] == :count
+        @count = true
+      end
+
+      @resource = @options.delete(:resource)
+      @resource ||= args.first
+      @resource ||= 'shop'
+      @resource = @resource.to_s
+
+      @id = @options.delete(:id)
+
+      @options[:limit] ||= SHOPIFY_API_MAX_LIMIT
+
+      resource_path = @resource
+      resource_path = resource_path.pluralize unless (@resource == 'shop')
+
+
+      path = [resource_path]
+      path.unshift('admin') unless Request.base_uri.path[-1] == '/' #handle quirk of URI.merge
+
+      @data_root = @resource
+      if @count
+        path << 'count'
+        @options.delete(:limit)
+        @data_root = 'count'
+      elsif @id
+        path << @id.to_s
+        @data_root = @resource
+      else
+        @data_root = @resource.pluralize
+      end
+
+      path = path.join('/') + '.json'
+      uri = Request.base_uri.merge(path)
+      uri.query = @options.to_param unless @options.empty?
+
+      super(uri.to_s)
+    end
+
+    def finish(response, bypass_memoization = nil)
+      response.extend Shydra::Response
+      super(response, bypass_memoization)
+    end
+
+  end  
+end
+
+

data/lib/shydra/response.rb

@@ -0,0 +1,15 @@
+require 'oj'
+module Shydra
+  module Response
+    def data
+      json = Oj.load(response_body)
+      case response_code
+      when 200
+        json[request.data_root]
+      else
+        json
+      end
+    end
+  end
+end
+ 

data/lib/shydra/version.rb

@@ -0,0 +1,3 @@
+module Shydra
+  VERSION = "0.0.1"
+end

data/shydra.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'shydra/version'
+Gem::Specification.new do |spec|
+  spec.name          = "shydra"
+  spec.version       = Shydra::VERSION
+  spec.authors       = ["Michael Johnston"]
+  spec.email         = ["lastobelus@mac.com"]
+  spec.description   = %q{A fast, parallel shopify api client using typhoeus/hydra. Attempts to balance being a good citizen about api limits and fetching multiple apai records in parallel. }
+  spec.summary       = %q{A fast, parallel shopify api client using typhoeus/hydraa}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+
+  spec.add_runtime_dependency "typhoeus", ">= 0.6.3"
+  spec.add_runtime_dependency "shopify_api", ">= 3.0.3"
+  spec.add_runtime_dependency "oj"
+  spec.add_development_dependency "bundler", ">= 1.3"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec"
+end

data/spec/shydra/hydra_spec.rb

@@ -0,0 +1,5 @@
+require 'spec_helper'
+
+describe "Hydra" do
+  it "needs to have tests written"
+end

data/spec/shydra/request_spec.rb

@@ -0,0 +1,80 @@
+require 'spec_helper'
+
+require 'shydra/request'
+
+
+describe "Shydra::Request" do
+  before do
+     ShopifyAPI::Base.stub(:site).and_return(URI("https://xxxx:yyyy@cronin.myshopify.com/admin/"))
+  end
+
+  describe "URIs" do
+    subject{ Shydra::Request.new() }
+    let(:store_uri){ URI("https://xxxx:yyyy@cronin.myshopify.com/admin/")}
+
+    specify{ expect(subject).to be_a_kind_of(Typhoeus::Request)}
+
+    it "gets the base url from Shopify API" do
+      expect(Shydra::Request.base_uri).to eq(store_uri)
+    end
+
+    it "creates resource collection uri" do
+      expect(Shydra::Request.new(:product).url).to eq(
+        store_uri.to_s + "products.json?limit=250")
+    end
+
+    it "creates resource collection uri with params" do
+      expect(Shydra::Request.new(:product, collection_id: 789, vendor: 'bob').url).to eq(
+        store_uri.to_s + "products.json?collection_id=789&limit=250&vendor=bob")
+    end
+
+    it "sets the resource to shop by default" do
+      expect(Shydra::Request.new.url).to eq(
+        store_uri.to_s + "shop.json?limit=250")
+    end
+
+    it "creates resource by id uri" do
+      expect(Shydra::Request.new(:product, id: 12345).url).to eq(
+        store_uri.to_s + "products/12345.json?limit=250")
+    end
+
+    it "creates resource count uri" do
+      expect(Shydra::Request.new(:product, :count).url).to eq(
+        store_uri.to_s + "products/count.json")
+    end
+
+    it "creates resource count uri with params" do
+      expect(Shydra::Request.new(:product, :count, collection_id: 789, vendor: 'bob').url).to eq(
+        store_uri.to_s + "products/count.json?collection_id=789&vendor=bob")
+    end
+  end
+
+  describe 'responses' do
+    describe '#data' do
+      it "returns the count as an int for a count request" do
+        response = Typhoeus::Response.new(code: 200, body: "{\"count\":14}")
+        Typhoeus.stub(/cronin/).and_return(response)
+        response = Shydra::Request.new(:product, :count).run
+
+        expect(response.data).to eq 14
+      end
+
+      it "returns a hash of product attributes for an id request" do
+        response = Typhoeus::Response.new(code: 200, body: "{\"product\":{\"body_html\":\"yo yo yo baby pop\",\"id\":137632345}}")
+        Typhoeus.stub(/cronin/).and_return(response)
+        response = Shydra::Request.new(:product, id: 137632345).run
+
+        expect(response.data).to eq( {"body_html"=>"yo yo yo baby pop", "id"=>137632345})
+      end
+
+      it "returns an array of hashes of product attributes for an collection request" do
+        response = Typhoeus::Response.new(code: 200, body: "{\"products\":[{\"body_html\":\"yo yo yo baby pop\",\"id\":123},{\"body_html\":\"yowza\",\"id\":456}]}")
+        Typhoeus.stub(/cronin/).and_return(response)
+        response = Shydra::Request.new(:product).run
+
+        expect(response.data).to eq( [{"body_html"=>"yo yo yo baby pop", "id"=>123}, {"body_html"=>"yowza", "id"=>456}] )
+      end
+    end
+  end
+
+end

data/spec/shydra/response_spec.rb

@@ -0,0 +1,5 @@
+require 'spec_helper'
+
+describe "Response" do
+  it "needs to have tests written"
+end

data/spec/shydra_spec.rb

@@ -0,0 +1,5 @@
+require 'spec_helper'
+
+describe "Shydra" do
+  it "needs to have tests written"
+end

data/spec/spec_helper.rb

@@ -0,0 +1,4 @@
+require 'rubygems'
+require 'bundler/setup'
+
+require 'shydra'

metadata

@@ -0,0 +1,172 @@
+--- !ruby/object:Gem::Specification
+name: shydra
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Michael Johnston
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-06-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: typhoeus
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.6.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.6.3
+- !ruby/object:Gem::Dependency
+  name: shopify_api
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 3.0.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 3.0.3
+- !ruby/object:Gem::Dependency
+  name: oj
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: ! 'A fast, parallel shopify api client using typhoeus/hydra. Attempts
+  to balance being a good citizen about api limits and fetching multiple apai records
+  in parallel. '
+email:
+- lastobelus@mac.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- Guardfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/shydra.rb
+- lib/shydra/hydra.rb
+- lib/shydra/request.rb
+- lib/shydra/response.rb
+- lib/shydra/version.rb
+- shydra.gemspec
+- spec/shydra/hydra_spec.rb
+- spec/shydra/request_spec.rb
+- spec/shydra/response_spec.rb
+- spec/shydra_spec.rb
+- spec/spec_helper.rb
+homepage: ''
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: 1355709900760715495
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: 1355709900760715495
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.25
+signing_key: 
+specification_version: 3
+summary: A fast, parallel shopify api client using typhoeus/hydraa
+test_files:
+- spec/shydra/hydra_spec.rb
+- spec/shydra/request_spec.rb
+- spec/shydra/response_spec.rb
+- spec/shydra_spec.rb
+- spec/spec_helper.rb