bucket_cake 0.2.0 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9f110ceec832023ab706c978ccc006509d2796ff
-  data.tar.gz: 13399289c80e169fb230dedbdb15fc7250a491a1
+  metadata.gz: e31f6c9d7a53111c74ea5c965aa98c2bfbd10ef3
+  data.tar.gz: 152f26894f635dd2c05d1990f79d7da47bacfcb4
 SHA512:
-  metadata.gz: ae786997482e3be5e536f3657e3335936a82ffc12bd775db4356f926608ced717ae6eceb7f5751d430dff6c41e0850eed69af91e28ab1eb4288caad5d3af9b0b
-  data.tar.gz: f0e6a0565fea804e9c3c36a9f367f60ef467eb0a4b6288be24abfadbb6a6dd45ea284fbd2f76dc7ea54188b23b79703749b7b9d3eb6da7a3acc9be1ad39785ba
+  metadata.gz: fd917f1e01d53d216d94baf41a5ed173a2e0cb044ead85d8e16982faf1cef6f30bb443343569925601882a7d70b848e1035a0e5637f90420ead85748170cfa0d
+  data.tar.gz: f244945c1ff6a96f8a7de5b90c4f9ff6274c22f0f35b1e3c5a3148faa7a88f4aa1c3496e554827a94fe69520f32065033d1c15ca64f4c19a9b9991a623c47fcb

data/lib/bucket_cake.rb

@@ -4,7 +4,9 @@ require 'protobuf'
 require 'active_support/time'
 
 require 'bucket_cake/version'
-require 'bucket_cake/stream'
+require 'bucket_cake/source'
+require 'bucket_cake/decoder'
+require 'bucket_cake/base'
 
 require 'bucket_cake/proto/common.pb'
 require 'bucket_cake/proto/advertisers.pb'
@@ -12,10 +14,19 @@ require 'bucket_cake/proto/affiliates.pb'
 require 'bucket_cake/proto/campaigns.pb'
 require 'bucket_cake/proto/creatives.pb'
 require 'bucket_cake/proto/offers.pb'
+require 'bucket_cake/proto/clicks.pb'
+require 'bucket_cake/proto/conversions.pb'
+require 'bucket_cake/proto/conversion_changes.pb'
+require 'bucket_cake/proto/cap_states.pb'
 
 require 'bucket_cake/proto_ext/time_with_zone'
 
 require 'bucket_cake/entities'
 require 'bucket_cake/values'
+require 'bucket_cake/facts'
 
 Protobuf.ignore_unknown_fields = false
+
+module BucketCake
+  CACHE_PREFIX = 'bucket-cake-latest'.freeze
+end

data/lib/bucket_cake/base.rb

@@ -0,0 +1,35 @@
+module BucketCake
+  class Base
+    def initialize(cache)
+      @cache = cache
+      check_cache!
+    end
+
+    private
+
+    attr_reader :cache
+
+    def check_cache!
+      return if cache && cache.respond_to?(:read) && cache.respond_to?(:write)
+
+      fail 'BucketCake: cache parameter must respond to #read and #write. ' \
+        'Use Rails.cache if you are not sure.'
+    end
+
+    def latest(type, klass, &b)
+      get Source::Latest.new(type, cache), klass, &b
+    end
+
+    def range(type, klass, &b)
+      get Source::Range.new(type, cache), klass, &b
+    end
+
+    def get(source, klass)
+      return unless source.new_data?
+
+      result = yield Decoder.new(source.zip_files, klass).items
+      source.confirm!
+      result
+    end
+  end
+end

data/lib/bucket_cake/decoder.rb

@@ -0,0 +1,30 @@
+module BucketCake
+  class Decoder
+    def initialize(zip_files, klass)
+      @zip_files = zip_files
+      @klass = klass
+    end
+
+    def items
+      Enumerator.new do |y|
+        zip_files.each do |zip_data|
+          unzip_elements(zip_data) do |element|
+            y << klass.decode(element)
+          end
+        end
+      end
+    end
+
+    private
+
+    attr_reader :zip_files, :klass
+
+    def unzip_elements(zip_data)
+      Zip::File.open_buffer(zip_data) do |files|
+        files.each do |entry|
+          yield entry.get_input_stream.read
+        end
+      end
+    end
+  end
+end

data/lib/bucket_cake/entities.rb

@@ -1,23 +1,23 @@
 module BucketCake
-  class Entities
-    def advertisers
-      Stream.new('advertisers', Cakeproto::Advertiser).get
+  class Entities < Base
+    def advertisers(&b)
+      latest 'advertisers', Cakeproto::Advertiser, &b
     end
 
-    def affiliates
-      Stream.new('affiliates', Cakeproto::Affiliate).get
+    def affiliates(&b)
+      latest 'affiliates', Cakeproto::Affiliate, &b
     end
 
-    def campaigns
-      Stream.new('campaigns', Cakeproto::Campaign).get
+    def campaigns(&b)
+      latest 'campaigns', Cakeproto::Campaign, &b
     end
 
-    def creatives
-      Stream.new('creatives', Cakeproto::Creative).get
+    def creatives(&b)
+      latest 'creatives', Cakeproto::Creative, &b
     end
 
-    def offers
-      Stream.new('offers', Cakeproto::Offer).get
+    def offers(&b)
+      latest 'offers', Cakeproto::Offer, &b
     end
   end
 end

data/lib/bucket_cake/facts.rb

@@ -0,0 +1,19 @@
+module BucketCake
+  class Facts < Base
+    def clicks(&b)
+      range 'clicks', Cakeproto::Click, &b
+    end
+
+    def conversions(&b)
+      range 'conversions', Cakeproto::Conversion, &b
+    end
+
+    def conversion_changes(&b)
+      range 'conversion_changes', Cakeproto::ConversionChange, &b
+    end
+
+    def cap_states(&b)
+      range 'cap_states', Cakeproto::CapState, &b
+    end
+  end
+end

data/lib/bucket_cake/proto/cap_states.pb.rb

@@ -0,0 +1,40 @@
+# encoding: utf-8
+
+##
+# This file is auto-generated. DO NOT EDIT!
+#
+
+
+##
+# Imports
+#
+
+module Cakeproto
+
+  ##
+  # Message Classes
+  #
+  class CapState < ::Protobuf::Message; end
+
+
+  ##
+  # Message Fields
+  #
+  class CapState
+    optional ::Cakeproto::IdName, :advertiser, 1
+    optional ::Cakeproto::IdName, :offer, 2
+    optional ::Cakeproto::IdName, :offer_contract, 3
+    optional ::Cakeproto::IdName, :affiliate, 4
+    optional :int64, :campaign_id, 5
+    optional :string, :entity, 6
+    optional ::Cakeproto::IdName, :type, 7
+    optional ::Cakeproto::IdName, :interval, 8
+    optional :bool, :active, 9
+    optional ::Cakeproto::TimeWithZone, :start_date, 10
+    optional ::Cakeproto::TimeWithZone, :end_date, 11
+    optional :int64, :current, 12
+    optional :int64, :limit, 13
+  end
+
+end
+

data/lib/bucket_cake/proto/clicks.pb.rb

@@ -0,0 +1,52 @@
+# encoding: utf-8
+
+##
+# This file is auto-generated. DO NOT EDIT!
+#
+
+
+##
+# Imports
+#
+
+module Cakeproto
+
+  ##
+  # Message Classes
+  #
+  class Click < ::Protobuf::Message; end
+
+
+  ##
+  # Message Fields
+  #
+  class Click
+    optional :int64, :id, 1
+    optional :int64, :visitor_id, 2
+    optional :int64, :request_session_id, 3
+    optional ::Cakeproto::TimeWithZone, :click_date, 4
+    optional ::Cakeproto::IdName, :affiliate, 5
+    optional ::Cakeproto::IdName, :advertiser, 6
+    optional ::Cakeproto::IdName, :offer, 7
+    optional ::Cakeproto::IdName, :offer_contract, 8
+    optional :int64, :campaign_id, 9
+    optional ::Cakeproto::IdName, :creative, 10
+    optional :string, :sub_id1, 11
+    optional :string, :sub_id2, 12
+    optional :string, :sub_id3, 13
+    optional :string, :sub_id4, 14
+    optional :string, :sub_id5, 15
+    optional :string, :ip_address, 16
+    optional :string, :user_agent, 17
+    optional :string, :referrer_url, 18
+    optional :string, :request_url, 19
+    optional :string, :redirect_url, 20
+    optional ::Cakeproto::Country, :country, 21
+    optional ::Cakeproto::Region, :region, 22
+    optional ::Cakeproto::IdName, :isp, 23
+    optional :string, :disposition, 24
+    optional :int64, :paid_action, 25
+  end
+
+end
+

data/lib/bucket_cake/proto/common.pb.rb

@@ -14,8 +14,10 @@ module Cakeproto
   class Currency < ::Protobuf::Message; end
   class PixelInfo < ::Protobuf::Message; end
   class Money < ::Protobuf::Message; end
+  class MoneyWithCurrency < ::Protobuf::Message; end
   class MediaType < ::Protobuf::Message; end
   class Country < ::Protobuf::Message; end
+  class Region < ::Protobuf::Message; end
   class SuppressionList < ::Protobuf::Message; end
   class Contact < ::Protobuf::Message; end
   class Blacklist < ::Protobuf::Message; end
@@ -24,6 +26,7 @@ module Cakeproto
   class EventOverride < ::Protobuf::Message; end
   class Upsell < ::Protobuf::Message; end
   class ExchangeRate < ::Protobuf::Message; end
+  class Disposition < ::Protobuf::Message; end
 
 
   ##
@@ -59,6 +62,12 @@ module Cakeproto
     optional :string, :formatted_amount, 3
   end
 
+  class MoneyWithCurrency
+    optional :int64, :currency_id, 1
+    optional :double, :amount, 2
+    optional :string, :formatted_amount, 3
+  end
+
   class MediaType
     optional :int64, :id, 1
     optional :string, :name, 2
@@ -71,6 +80,11 @@ module Cakeproto
     optional :string, :name, 2
   end
 
+  class Region
+    optional :string, :code, 1
+    optional :string, :name, 2
+  end
+
   class SuppressionList
     optional :int64, :id, 1
     optional :string, :name, 2
@@ -150,5 +164,13 @@ module Cakeproto
     optional :double, :rate, 5
   end
 
+  class Disposition
+    optional :int64, :id, 1
+    optional :string, :name, 2
+    optional ::Cakeproto::IdName, :type, 3
+    optional :string, :contact, 4
+    optional ::Cakeproto::TimeWithZone, :disposition_date, 5
+  end
+
 end
 

data/lib/bucket_cake/proto/conversion_changes.pb.rb

@@ -0,0 +1,34 @@
+# encoding: utf-8
+
+##
+# This file is auto-generated. DO NOT EDIT!
+#
+
+
+##
+# Imports
+#
+
+module Cakeproto
+
+  ##
+  # Message Classes
+  #
+  class ConversionChange < ::Protobuf::Message; end
+
+
+  ##
+  # Message Fields
+  #
+  class ConversionChange
+    optional :int64, :id, 1
+    optional ::Cakeproto::TimeWithZone, :conversion_date, 2
+    optional ::Cakeproto::TimeWithZone, :last_updated, 3
+    optional ::Cakeproto::MoneyWithCurrency, :paid, 4
+    optional ::Cakeproto::MoneyWithCurrency, :received, 5
+    optional ::Cakeproto::Disposition, :current_disposition, 6
+    optional :string, :note, 7
+  end
+
+end
+

data/lib/bucket_cake/proto/conversions.pb.rb

@@ -0,0 +1,65 @@
+# encoding: utf-8
+
+##
+# This file is auto-generated. DO NOT EDIT!
+#
+
+
+##
+# Imports
+#
+
+module Cakeproto
+
+  ##
+  # Message Classes
+  #
+  class Conversion < ::Protobuf::Message; end
+
+
+  ##
+  # Message Fields
+  #
+  class Conversion
+    optional :int64, :id, 1
+    optional :int64, :visitor_id, 2
+    optional :int64, :request_session_id, 3
+    optional :int64, :click_id, 4
+    optional :int64, :click_request_session_id, 5
+    optional ::Cakeproto::TimeWithZone, :conversion_date, 6
+    optional ::Cakeproto::TimeWithZone, :click_date, 7
+    optional ::Cakeproto::TimeWithZone, :last_updated, 8
+    optional ::Cakeproto::IdName, :event, 9
+    optional ::Cakeproto::IdName, :affiliate, 10
+    optional ::Cakeproto::IdName, :advertiser, 11
+    optional ::Cakeproto::IdName, :offer, 12
+    optional ::Cakeproto::IdName, :offer_contract, 13
+    optional :int64, :campaign_id, 14
+    optional ::Cakeproto::IdName, :campaign_type, 15
+    optional ::Cakeproto::IdName, :creative, 16
+    optional :string, :sub_id1, 17
+    optional :string, :sub_id2, 18
+    optional :string, :sub_id3, 19
+    optional :string, :sub_id4, 20
+    optional :string, :sub_id5, 21
+    optional :string, :ip_address, 22
+    optional :string, :click_ip_address, 36
+    optional :string, :user_agent, 23
+    optional :string, :click_user_agent, 37
+    optional :string, :source_type, 24
+    optional ::Cakeproto::IdName, :price_format, 25
+    optional ::Cakeproto::MoneyWithCurrency, :paid, 26
+    optional ::Cakeproto::MoneyWithCurrency, :received, 27
+    optional :int64, :step_reached, 28
+    optional :string, :transaction_id, 29
+    optional ::Cakeproto::Disposition, :current_disposition, 30
+    optional ::Cakeproto::MoneyWithCurrency, :order_total, 38
+    optional ::Cakeproto::Country, :country, 31
+    optional ::Cakeproto::Region, :region, 32
+    optional ::Cakeproto::IdName, :language, 33
+    optional ::Cakeproto::IdName, :isp, 34
+    optional :string, :note, 35
+  end
+
+end
+

data/lib/bucket_cake/source.rb

@@ -0,0 +1,83 @@
+module BucketCake
+  class Source
+    def initialize(type, cache)
+      @type = type
+      @cache = cache
+    end
+
+    def zip_files
+      Enumerator.new do |y|
+        objects.each do |object|
+          next unless check_success(object.key)
+          next unless object.key =~ %r{/part\d{3}\.zip\z}
+
+          io = object.get.body
+
+          # we need to force binary encoding, otherwise binary zip data will be parsed as utf-8
+          io.set_encoding('BINARY'.freeze)
+
+          y << io.read
+        end
+      end
+    end
+
+    def check_success(key)
+      @success_files ||= Set.new
+      @success_files << key if key.end_with?('/SUCCESS'.freeze)
+
+      # This relies on the fact that SUCCESS files always come before partNNN.zip files.
+      # But that is fine because S3 guarantees alphabetical order.
+      @success_files.include?(File.dirname(key) + '/SUCCESS'.freeze)
+    end
+
+    def new_data?
+      current_latest_path != cache_latest_path
+    end
+
+    def confirm!
+      cache.write(cache_key, current_latest_path)
+    end
+
+    private
+
+    attr_reader :type, :cache
+
+    def cache_key
+      "#{CACHE_PREFIX}-#{type}"
+    end
+
+    def current_latest_path
+      @current_latest_path ||= bucket.object("#{type}/latest").get.body.read
+    end
+
+    def cache_latest_path
+      @cache_latest_path ||= cache.read(cache_key)
+    end
+
+    def bucket
+      @bucket ||= Aws::S3::Bucket.new(ENV.fetch('CAKE_DATA_BUCKET'.freeze))
+    end
+  end
+
+  class Source::Latest < Source
+    private
+
+    def objects
+      bucket.objects(prefix: "#{type}/#{current_latest_path}/")
+    end
+  end
+
+  class Source::Range < Source
+    private
+
+    def objects
+      # If we don't have a timestamp in cache, return everything
+      return bucket.objects(prefix: "#{type}/") if cache_latest_path.nil?
+
+      # This returns all objects after the current timestamp.
+      # By using the 'xxx' suffix, none of the objects of the given timestamp are returned.
+      # S3 ensures alphabetical order of the results, so this works fine for date ranges.
+      bucket.objects(marker: "#{type}/#{cache_latest_path}/xxx", prefix: "#{type}/")
+    end
+  end
+end

data/lib/bucket_cake/values.rb

@@ -1,35 +1,35 @@
 module BucketCake
-  class Values
-    def affiliate_tiers
-      Stream.new('affiliate_tiers', Cakeproto::IdName).get
+  class Values < Base
+    def affiliate_tiers(&b)
+      latest 'affiliate_tiers', Cakeproto::IdName, &b
     end
 
-    def blacklist_reasons
-      Stream.new('blacklist_reasons', Cakeproto::IdName).get
+    def blacklist_reasons(&b)
+      latest 'blacklist_reasons', Cakeproto::IdName, &b
     end
 
-    def countries
-      Stream.new('countries', Cakeproto::Country).get
+    def countries(&b)
+      latest 'countries', Cakeproto::Country, &b
     end
 
-    def currencies
-      Stream.new('currencies', Cakeproto::Currency).get
+    def currencies(&b)
+      latest 'currencies', Cakeproto::Currency, &b
     end
 
-    def exchange_rates
-      Stream.new('exchange_rates', Cakeproto::ExchangeRate).get
+    def exchange_rates(&b)
+      latest 'exchange_rates', Cakeproto::ExchangeRate, &b
     end
 
-    def media_types
-      Stream.new('media_types', Cakeproto::IdName).get
+    def media_types(&b)
+      latest 'media_types', Cakeproto::IdName, &b
     end
 
-    def payment_types
-      Stream.new('payment_types', Cakeproto::IdName).get
+    def payment_types(&b)
+      latest 'payment_types', Cakeproto::IdName, &b
     end
 
-    def verticals
-      Stream.new('verticals', Cakeproto::IdName).get
+    def verticals(&b)
+      latest 'verticals', Cakeproto::IdName, &b
     end
   end
 end

data/lib/bucket_cake/version.rb

@@ -1,3 +1,3 @@
 module BucketCake
-  VERSION = '0.2.0'.freeze
+  VERSION = '1.0.0'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bucket_cake
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 1.0.0
 platform: ruby
 authors:
 - ad2games developers
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-12-02 00:00:00.000000000 Z
+date: 2015-12-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk
@@ -172,15 +172,22 @@ extensions: []
 extra_rdoc_files: []
 files:
 - lib/bucket_cake.rb
+- lib/bucket_cake/base.rb
+- lib/bucket_cake/decoder.rb
 - lib/bucket_cake/entities.rb
+- lib/bucket_cake/facts.rb
 - lib/bucket_cake/proto/advertisers.pb.rb
 - lib/bucket_cake/proto/affiliates.pb.rb
 - lib/bucket_cake/proto/campaigns.pb.rb
+- lib/bucket_cake/proto/cap_states.pb.rb
+- lib/bucket_cake/proto/clicks.pb.rb
 - lib/bucket_cake/proto/common.pb.rb
+- lib/bucket_cake/proto/conversion_changes.pb.rb
+- lib/bucket_cake/proto/conversions.pb.rb
 - lib/bucket_cake/proto/creatives.pb.rb
 - lib/bucket_cake/proto/offers.pb.rb
 - lib/bucket_cake/proto_ext/time_with_zone.rb
-- lib/bucket_cake/stream.rb
+- lib/bucket_cake/source.rb
 - lib/bucket_cake/values.rb
 - lib/bucket_cake/version.rb
 homepage: https://github.com/ad2games/bucket_cake
@@ -203,7 +210,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.0
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: Get cakeproto objects from S3.

data/lib/bucket_cake/stream.rb

@@ -1,45 +0,0 @@
-module BucketCake
-  class Stream
-    def initialize(type, klass)
-      @type = type
-      @klass = klass
-    end
-
-    def get
-      Enumerator.new do |y|
-        objects.each do |object|
-          unzip_elements(object.get.body) do |element|
-            y << klass.decode(element)
-          end
-        end
-      end
-    end
-
-    private
-
-    attr_reader :type, :klass
-
-    def unzip_elements(zip_io)
-      # we need to force binary encoding, otherwise binary zip data will be parsed as utf-8
-      zip_io.set_encoding('BINARY'.freeze)
-
-      Zip::File.open_buffer(zip_io.read) do |files|
-        files.each do |entry|
-          yield entry.get_input_stream.read
-        end
-      end
-    end
-
-    def objects
-      bucket.objects(prefix: "#{type}/#{latest_path}/part")
-    end
-
-    def latest_path
-      bucket.object("#{type}/latest").get.body.read
-    end
-
-    def bucket
-      @bucket ||= Aws::S3::Bucket.new(ENV.fetch('CAKE_DATA_BUCKET'.freeze))
-    end
-  end
-end