tap-vend 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 7dfef7b8aa2a9ba8b9402a3a4bd4bf0f53054d64
+  data.tar.gz: 6f050959e21002cfee3e29934661cc1a1fa77fa3
+SHA512:
+  metadata.gz: e84f324d907ec2cc9f97c4b0768885a3aec48d80a2696128946d3df774c89165ef4b3fb524a25b6f05fb195501e8754d09b904e33fcd069a05f74a34055ea941
+  data.tar.gz: a8ccdac5a0a1e32ddb6add6b5624e01c365a9fa126a218550143042262c0971371a87c6567e9dde854f6724d645b17c685878210354f61461f8aa5e0ec359617

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2017 follain
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,25 @@
+tap-vend
+========
+
+## A [singer.io](http://singer.io) tap to extract data from [Vend POS](http://vendhq.com) and load into any Singer target, like Stitch or CSV
+
+# Configuration
+
+    {
+      "store":"your-vend-store",
+      "token":"your-vend-personal-token",
+      "requests_per_second": 2
+    }
+
+# Usage (with [Stitch target](https://github.com/singer-io/target-stitch))
+
+    > bundle exec tap-vend
+    Usage: tap-vend [options]
+        -c, --config config_file         Set config file (json)
+        -s, --state state_file           Set state file (json)
+        -h, --help                       Displays help
+        -v, --verbose                    Enables verbose logging to STDERR
+
+    > pip install target-stitch
+    > gem install tap-vend
+    > bundle exec tap-vend -c config.vend.json -s state.json | target-stitch --config config.stitch.json | tail -1 > state.new.json

data/bin/tap-vend

@@ -0,0 +1,89 @@
+#!/usr/bin/env ruby
+
+require 'bundler'
+
+Bundler.require
+
+require 'concurrent'
+require 'faraday'
+require 'faraday_middleware'
+require 'optparse'
+require 'peach'
+
+require './lib/client'
+require './lib/schema'
+require './lib/models/base'
+
+require './lib/models/consignment'
+require './lib/models/customer'
+require './lib/models/customer_group'
+require './lib/models/inventory'
+require './lib/models/outlet'
+require './lib/models/outlet_attribute'
+require './lib/models/outlet_tax'
+require './lib/models/payment_type'
+require './lib/models/price_book'
+require './lib/models/price_book_product'
+require './lib/models/product'
+require './lib/models/product_type'
+require './lib/models/register'
+require './lib/models/sale'
+require './lib/models/supplier'
+require './lib/models/tax'
+require './lib/models/user'
+
+config_file = nil
+state_file = nil
+verbose = false
+
+parser = OptionParser.new do |opts|
+  opts.banner = "Usage: #{$0} [options]"
+  opts.on('-c', '--config config_file', 'Set config file (json)') do |config|
+    config_file = config
+  end
+
+  opts.on('-s', '--state state_file', 'Set state file (json)') do |state|
+    state_file = state
+  end
+
+  opts.on('-h', '--help', 'Displays help') do
+    puts opts
+    exit
+  end
+
+  opts.on('-v', '--verbose', 'Enables verbose logging to STDERR') do
+    verbose = true
+  end
+end
+
+parser.parse!
+
+if config_file.nil?
+  puts parser
+  exit
+end
+
+config = JSON.parse(File.read(config_file))
+
+state = {}
+if state_file
+  state = JSON.parse(File.read(state_file))
+end
+
+
+Client.requests_per_second = config['requests_per_second']
+
+client = Client.new(
+  config['store'],
+  config['token'],
+  verbose,
+  state
+)
+
+Models::Base.subclasses.each do |model|
+  client.output model.schema
+end
+
+Models::Base.subclasses.peach(50) do |model|
+  client.process model
+end

data/lib/client.rb

@@ -0,0 +1,72 @@
+require 'json'
+require 'logger'
+require 'thread'
+require 'glutton_ratelimit'
+
+Client = Struct.new(:store, :token, :verbose, :state) do
+  extend GluttonRatelimit
+  MUTEX = Mutex.new
+  OUTPUT = Mutex.new
+
+  def self.requests_per_second=(rps)
+    rps = rps.to_r
+    if rps > 0
+      rate_limit :get, rps.numerator, rps.denominator
+    end
+  end
+
+  def process(model, after = 0)
+    MUTEX.synchronize do
+      self.state ||= Concurrent::Hash.new()
+    end
+
+    state['value'] ||= Concurrent::Hash.new()
+
+    return unless model.path
+    after = state['value'][model.stream].to_i
+
+    records = get(model.path, after: after)
+
+    if Array(records['data']).flatten.compact.any?
+      records['data'].peach(50) do |record|
+        model.new(record, self).records.flatten.each do |model_record|
+          output model_record
+        end
+      end
+      state['value'][model.stream] = records['version']['max']
+    end
+
+    output(type: :STATE, value: state)
+  end
+
+  def output(object)
+    OUTPUT.synchronize do
+      puts JSON.generate(object)
+    end
+  end
+
+  def get(path, after: 0, page_size: 5000)
+    connection.get("/api/2.0/#{path}", after: after, page_size: page_size).body
+  end
+
+  private
+
+  def log(string)
+    OUTPUT.synchronize do
+      (@logger ||= ::Logger.new(STDERR)).info string
+    end
+  end
+
+  def connection
+    MUTEX.synchronize do
+      @connection ||= Faraday::Connection.new do |conn|
+        conn.authorization :Bearer, token
+        conn.response :json
+        conn.url_prefix = "https://#{store}.vendhq.com"
+        conn.response :logger, ::Logger.new(STDERR) if verbose
+        conn.adapter Faraday.default_adapter
+      end
+    end
+    @connection
+  end
+end

data/lib/models/base.rb

@@ -0,0 +1,43 @@
+require 'active_support/inflector'
+
+module Models
+  Base = Struct.new(:data, :client) do
+    def self.subclasses
+      ObjectSpace.each_object(Class).select { |klass| klass < self }
+    end
+
+    def self.path
+      name.demodulize.tableize
+    end
+
+    def self.stream
+      name.demodulize.tableize
+    end
+
+    def self.schema(&block)
+      @schema ||= Schema.new(stream)
+      @schema.instance_eval &block if block_given?
+      @schema.to_hash
+    end
+
+    def transform
+      data.dup
+    end
+
+    def base_record
+      {
+        type: "RECORD",
+        stream: self.class.stream,
+        record: transform
+      }
+    end
+
+    def extra_records
+      []
+    end
+
+    def records
+      [ base_record ] + extra_records.map(&:records)
+    end
+  end
+end

data/lib/models/consignment.rb

@@ -0,0 +1,38 @@
+
+module Models
+  class ConsignmentProduct < Base
+    def self.path
+      nil
+    end
+
+    schema do
+      string :id
+    end
+
+    def transform
+      super.tap do |hash|
+        hash.merge!(
+          'id' =>
+             hash.slice('consignment_id', 'product_id').values.join('.')
+        )
+      end
+    end
+  end
+
+  class Consignment < Base
+    schema do
+      string :id
+    end
+
+    def extra_records
+      return [] unless data['deleted_at'].nil?
+      products = client.get("consignments/#{data['id']}/products")['data']
+      Array(products).flatten.map do |product|
+        ConsignmentProduct.new(
+          product.merge('consignment_id' => data['id']),
+          client
+        )
+      end
+    end
+  end
+end

data/lib/models/customer.rb

@@ -0,0 +1,8 @@
+
+module Models
+  class Customer < Base
+    schema do
+      string :id
+    end
+  end
+end

data/lib/models/customer_group.rb

@@ -0,0 +1,8 @@
+
+module Models
+  class CustomerGroup < Base
+    schema do
+      string :id
+    end
+  end
+end

data/lib/models/inventory.rb

@@ -0,0 +1,12 @@
+
+module Models
+  class Inventory < Base
+    def self.path
+      'inventory'
+    end
+
+    schema do
+      string :id
+    end
+  end
+end

data/lib/models/outlet.rb

@@ -0,0 +1,31 @@
+
+module Models
+  class Outlet < Base
+    schema do
+      string :id
+      string :name
+      string :physical_address_1
+      string :physical_address_2
+      string :physical_suburb
+      string :physical_city
+      string :physical_postcode
+      string :physical_state
+      string :physical_country_id
+    end
+
+    def transform
+      super.tap do |hash|
+        hash.delete 'attributes'
+      end
+    end
+
+    def extra_records
+      data['attributes'].map do |attribute|
+        OutletAttribute.new(
+          attribute.merge('outlet_id' => data['id']),
+          client
+        )
+      end
+    end
+  end
+end

data/lib/models/outlet_attribute.rb

@@ -0,0 +1,21 @@
+
+module Models
+  class OutletAttribute < Base
+    def self.path
+      nil
+    end
+
+    schema do
+      string :id
+      string :outlet_id
+      string :key
+      string :value
+    end
+
+    def transform
+      super.tap do |hash|
+        hash.merge! 'id' => hash.slice('outlet_id', 'key').values.join('.')
+      end
+    end
+  end
+end

data/lib/models/outlet_tax.rb

@@ -0,0 +1,17 @@
+
+module Models
+  class OutletTax < Base
+    schema do
+      string :id
+    end
+
+    def transform
+      super.tap do |hash|
+        hash.merge!(
+          'id' =>
+             hash.slice('outlet_id', 'product_id', 'tax_id').values.join('.')
+        )
+      end
+    end
+  end
+end

data/lib/models/payment_type.rb

@@ -0,0 +1,8 @@
+
+module Models
+  class PaymentType < Base
+    schema do
+      string :id
+    end
+  end
+end

data/lib/models/price_book.rb

@@ -0,0 +1,8 @@
+
+module Models
+  class PriceBook < Base
+    schema do
+      string :id
+    end
+  end
+end

data/lib/models/price_book_product.rb

@@ -0,0 +1,8 @@
+
+module Models
+  class PriceBookProduct < Base
+    schema do
+      string :id
+    end
+  end
+end

data/lib/models/product.rb

@@ -0,0 +1,8 @@
+
+module Models
+  class Product < Base
+    schema do
+      string :id
+    end
+  end
+end

data/lib/models/product_type.rb

@@ -0,0 +1,8 @@
+
+module Models
+  class ProductType < Base
+    schema do
+      string :id
+    end
+  end
+end

data/lib/models/register.rb

@@ -0,0 +1,18 @@
+
+module Models
+  class Register < Base
+    schema do
+      string :id
+      string :name
+      string :outlet_id
+      string :invoice_prefix
+      string :invoice_suffix
+    end
+
+    def transform
+      super.tap do |hash|
+        hash.delete 'attributes'
+      end
+    end
+  end
+end

data/lib/models/sale.rb

@@ -0,0 +1,8 @@
+
+module Models
+  class Sale < Base
+    schema do
+      string :id
+    end
+  end
+end

data/lib/models/supplier.rb

@@ -0,0 +1,8 @@
+
+module Models
+  class Supplier < Base
+    schema do
+      string :id
+    end
+  end
+end

data/lib/models/tax.rb

@@ -0,0 +1,8 @@
+
+module Models
+  class Tax < Base
+    schema do
+      string :id
+    end
+  end
+end

data/lib/models/user.rb

@@ -0,0 +1,28 @@
+
+module Models
+  class User < Base
+    schema do
+      string :id
+      string :username
+      string :displayname
+      string :email, nil: true
+      string :email_verified_at, nil: true
+      string :account_type
+      string :created_at
+      string :updated_at
+      string :deleted_at, nil: true
+    end
+
+    def transform
+      super.slice 'id',
+        'username',
+        'email',
+        'account_type',
+        'email_verified_at',
+        'created_at',
+        'updated_at',
+        'deleted_at',
+        'seen_at'
+    end
+  end
+end

data/lib/schema.rb

@@ -0,0 +1,25 @@
+
+Schema = Struct.new(:stream) do
+  def string(name, options={})
+    if options[:nil]
+      properties[name.to_sym] = { type: [:string, :null] }
+    else
+      properties[name.to_sym] = { type: :string }
+    end
+  end
+
+  def properties
+    @properties ||= {}
+  end
+
+  def to_hash
+    {
+      type: :SCHEMA,
+      stream: stream,
+      key_properties: [:id],
+      schema: {
+        properties: properties
+      }
+    }
+  end
+end

data/lib/tap_vend/version.rb

@@ -0,0 +1,3 @@
+module TapVend
+  VERSION = '0.0.1'
+end

metadata

@@ -0,0 +1,188 @@
+--- !ruby/object:Gem::Specification
+name: tap-vend
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Joe Lind
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-06-21 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 5.1.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 5.1.1
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.2
+- !ruby/object:Gem::Dependency
+  name: faraday
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.12'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.12.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.12'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.12.1
+- !ruby/object:Gem::Dependency
+  name: faraday_middleware
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.11'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.11.0.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.11'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.11.0.1
+- !ruby/object:Gem::Dependency
+  name: glutton_ratelimit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.0
+- !ruby/object:Gem::Dependency
+  name: peach
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.5.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.5.1
+description: Stream Vend POS records to a Singer target, such as Stitch
+email: joelind@gmail.com
+executables:
+- tap-vend
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- README.md
+- bin/tap-vend
+- lib/client.rb
+- lib/models/base.rb
+- lib/models/consignment.rb
+- lib/models/customer.rb
+- lib/models/customer_group.rb
+- lib/models/inventory.rb
+- lib/models/outlet.rb
+- lib/models/outlet_attribute.rb
+- lib/models/outlet_tax.rb
+- lib/models/payment_type.rb
+- lib/models/price_book.rb
+- lib/models/price_book_product.rb
+- lib/models/product.rb
+- lib/models/product_type.rb
+- lib/models/register.rb
+- lib/models/sale.rb
+- lib/models/supplier.rb
+- lib/models/tax.rb
+- lib/models/user.rb
+- lib/schema.rb
+- lib/tap_vend/version.rb
+homepage: https://github.com/Follain/tap-vend
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.2
+signing_key: 
+specification_version: 4
+summary: Singer.io tap for Vend POS
+test_files: []