bucket_cake 5.2.1 → 6.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c70c6ebd3e74463c225cf381f435bd9ba555eb2a
-  data.tar.gz: 40d780925bac1c782eea33810d0c31662f0fee40
+  metadata.gz: 2883992442100adbcd532c41921f5054242b8c6c
+  data.tar.gz: 5e4d397a504c4a976b0eb5bbbbb27d8c7c83ed99
 SHA512:
-  metadata.gz: 1682f264a01266b295152ea1a66cf3f76266a5ae6edcf823ed08c4d0038fe7ac6300a33bb7a1f8419c80fe0ed42ea3945a9cee096ce37ca029c1c653d315efe9
-  data.tar.gz: cd5a860472517fdc2f0d2b80ce8fc665c2505b2a36615d9efbceb215d969c504c53a35a609f8bfd5ec2b7df300486b990dd961e4036fd885478f6cbb9144b9ad
+  metadata.gz: 7868a377f26766dd2416aab145894eb963f588ea92bafa3aaf5364110449af913af0017f7ebfca1b635f5bc0293b5f2c19e7a0096fff4fa4dc983896034d793e
+  data.tar.gz: f09f55f39ee24be7ac16e0ca99ba90a3242790fee052c03b0961c6a25b4c91862f06d68d5c87c652df840e21a7f4d4bc26285607c6295e38ac3efe6704b76792

data/lib/bucket_cake/base.rb

@@ -1,46 +1,91 @@
 # frozen_string_literal: true
 module BucketCake
   class Base
-    class Range < self
-      private
+    def items
+      klass = self.class::PROTOCLASS.call
+      Decoder.new(files, klass).items.lazy
+    end
 
-      def source
-        Source::Range.new(self.class::FOLDER, cursor)
+    private
+
+    def files
+      Enumerator.new do |y|
+        keys.each do |key|
+          y << bucket.object(key).get.body
+        end
       end
     end
 
-    class Latest < self
-      def initialize(cursor = nil)
-        super(cursor)
-      end
+    def folder
+      self.class::FOLDER
+    end
 
-      def has_new_data?
-        cursor.nil? || cursor != source.latest_cursor
+    def bucket
+      @bucket ||= Aws::S3::Bucket.new(ENV.fetch('CAKE_DATA_BUCKET'))
+    end
+
+    class Range < self
+      include TimeHelper
+
+      attr_reader :start_time, :end_time
+
+      def initialize(start_time, end_time)
+        assert_time(start_time)
+        assert_time(end_time)
+        raise 'Invalid time: end must be after start' unless end_time > start_time
+
+        @start_time = start_time
+        @end_time = end_time
       end
 
       private
 
-      def source
-        Source::Latest.new(self.class::FOLDER, cursor)
+      def keys
+        hour_keys(folder, start_time, end_time)
       end
     end
 
-    attr_reader :cursor
+    class Latest < self
+      LATEST_FILE = 'latest.gz'
 
-    def initialize(cursor)
-      raise 'BucketCake: cursor has invalid format!' unless cursor.nil? || cursor =~ CURSOR_REGEXP
-      @cursor = cursor
-    end
+      private
 
-    def items
-      get source, self.class::PROTOCLASS.call
+      def keys
+        ["#{folder}/#{LATEST_FILE}"]
+      end
     end
 
-    private
+    class Realtime < self
+      include TimeHelper
+
+      DEFAULT_LOOKBACK = 24.hours
+      CURSOR_REGEX = %r{\A\w+/\d{4}/\d{2}/\d{2}/\d{4}\.gz\z}
+
+      attr_reader :cursor
+
+      def initialize(cursor)
+        raise 'Invalid cursor format!' unless cursor.nil? || cursor =~ CURSOR_REGEX
+        @cursor = cursor || default_cursor
+      end
 
-    def get(source, klass)
-      @cursor = source.latest_cursor
-      Decoder.new(source.zip_files, klass).items.lazy
+      def files
+        @keys = load_keys
+        @cursor = keys.last unless keys.empty?
+        super
+      end
+
+      private
+
+      attr_reader :keys
+
+      def default_cursor
+        minute_cursor(folder, Time.now - DEFAULT_LOOKBACK)
+      end
+
+      def load_keys
+        # return all objects after the cursor timestamp
+        bucket.objects(marker: cursor, prefix: "#{folder}/").map(&:key)
+      end
     end
   end
 end

data/lib/bucket_cake/decoder.rb

@@ -1,15 +1,15 @@
 # frozen_string_literal: true
 module BucketCake
   class Decoder
-    def initialize(zip_files, klass)
-      @zip_files = zip_files
+    def initialize(files, klass)
+      @files = files
       @klass = klass
     end
 
     def items
       Enumerator.new do |y|
-        zip_files.each do |zip_data|
-          unzip_elements(zip_data) do |element|
+        files.each do |gzdata|
+          unpack(gzdata) do |element|
             y << klass.decode(element)
           end
         end
@@ -18,15 +18,14 @@ module BucketCake
 
     private
 
-    attr_reader :zip_files, :klass
+    attr_reader :files, :klass
 
-    def unzip_elements(zip_data)
-      zip_data_io = StringIO.new(zip_data)
-      files = Zip::File.new(zip_data_io, true, true)
-      files.read_from_stream(zip_data_io)
+    def unpack(gzdata)
+      Zlib::GzipReader.new(gzdata).each_line.each do |line|
+        line.strip!
+        next if line.empty?
 
-      files.each do |entry|
-        yield entry.get_input_stream.read
+        yield Base64.decode64(line)
       end
     end
   end

data/lib/bucket_cake/facts.rb

@@ -11,14 +11,9 @@ module BucketCake
       PROTOCLASS = -> { Cakeproto::Conversion }
     end
 
-    class CapStates < Base::Range
+    class CapStatesLatest < Base::Latest
       FOLDER = 'cap_states'
       PROTOCLASS = -> { Cakeproto::CapState }
-
-      class Latest < Base::Latest
-        FOLDER = 'cap_states'
-        PROTOCLASS = -> { Cakeproto::CapState }
-      end
     end
   end
 end

data/lib/bucket_cake/realtime.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+module BucketCake
+  module Realtime
+    class Clicks < Base::Realtime
+      FOLDER = 'clicks_rt'
+      PROTOCLASS = -> { Cakeproto::Click }
+    end
+
+    class Conversions < Base::Realtime
+      FOLDER = 'conversions_rt'
+      PROTOCLASS = -> { Cakeproto::Conversion }
+    end
+  end
+end

data/lib/bucket_cake/time_helper.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+module BucketCake
+  module TimeHelper
+    def hour_cursor(prefix, time)
+      "#{prefix}/#{time.utc.strftime('%Y/%m/%d/%H')}.gz"
+    end
+
+    def minute_cursor(prefix, time)
+      "#{prefix}/#{time.utc.strftime('%Y/%m/%d/%H%M')}.gz"
+    end
+
+    def hour_keys(prefix, start_time, end_time)
+      Enumerator.new do |y|
+        while start_time < end_time
+          y << hour_cursor(prefix, start_time)
+          start_time += 1.hour
+        end
+      end
+    end
+
+    def assert_time(time)
+      raise 'Invalid time: must be a Time object' unless time.is_a?(Time)
+      raise 'Invalid time: must be at start of hour' if time.beginning_of_hour != time
+    end
+  end
+end

data/lib/bucket_cake/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module BucketCake
-  VERSION = '5.2.1'
+  VERSION = '6.0.0'
 end

data/lib/bucket_cake.rb

@@ -1,11 +1,12 @@
 # frozen_string_literal: true
+require 'base64'
+require 'zlib'
 require 'aws-sdk'
-require 'zip'
 require 'google/protobuf'
 require 'active_support/time'
 
 require 'bucket_cake/version'
-require 'bucket_cake/source'
+require 'bucket_cake/time_helper'
 require 'bucket_cake/decoder'
 require 'bucket_cake/base'
 
@@ -25,7 +26,4 @@ require 'bucket_cake/proto_ext/maybe_int'
 require 'bucket_cake/entities'
 require 'bucket_cake/values'
 require 'bucket_cake/facts'
-
-module BucketCake
-  CURSOR_REGEXP = %r{\A\d{4}/\d{2}/\d{2}/\d{6}\z}
-end
+require 'bucket_cake/realtime'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bucket_cake
 version: !ruby/object:Gem::Version
-  version: 5.2.1
+  version: 6.0.0
 platform: ruby
 authors:
 - ad2games developers
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-12-09 00:00:00.000000000 Z
+date: 2016-12-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk
@@ -24,20 +24,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: rubyzip
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: google-protobuf
   requirement: !ruby/object:Gem::Requirement
@@ -187,7 +173,8 @@ files:
 - lib/bucket_cake/proto/offers_pb.rb
 - lib/bucket_cake/proto_ext/maybe_int.rb
 - lib/bucket_cake/proto_ext/time_with_zone.rb
-- lib/bucket_cake/source.rb
+- lib/bucket_cake/realtime.rb
+- lib/bucket_cake/time_helper.rb
 - lib/bucket_cake/values.rb
 - lib/bucket_cake/version.rb
 homepage: https://github.com/ad2games/bucket_cake

data/lib/bucket_cake/source.rb

@@ -1,68 +0,0 @@
-# frozen_string_literal: true
-module BucketCake
-  class Source
-    def initialize(folder, cursor)
-      @folder = folder
-      @cursor = cursor
-    end
-
-    def zip_files
-      Enumerator.new do |y|
-        objects.each do |object|
-          next unless check_success(object.key)
-          next unless object.key =~ %r{/part\d{3}\.zip\z}
-
-          io = object.get.body
-
-          # we need to force binary encoding, otherwise binary zip data will be parsed as utf-8
-          io.set_encoding('BINARY')
-
-          y << io.read
-        end
-      end
-    end
-
-    def latest_cursor
-      @latest_cursor ||= bucket.object("#{folder}/latest").get.body.read
-    end
-
-    private
-
-    attr_reader :folder, :cursor
-
-    def check_success(key)
-      @success_files ||= Set.new
-      @success_files << key if key.end_with?('/SUCCESS')
-
-      # This relies on the fact that SUCCESS files always come before partNNN.zip files.
-      # But that is fine because S3 guarantees alphabetical order.
-      @success_files.include?(File.dirname(key) + '/SUCCESS')
-    end
-
-    def bucket
-      @bucket ||= Aws::S3::Bucket.new(ENV.fetch('CAKE_DATA_BUCKET'))
-    end
-  end
-
-  class Source::Latest < Source
-    private
-
-    def objects
-      bucket.objects(prefix: "#{folder}/#{latest_cursor}/")
-    end
-  end
-
-  class Source::Range < Source
-    private
-
-    def objects
-      # If we don't have a timestamp in cursor, return everything
-      return bucket.objects(prefix: "#{folder}/") if cursor.nil?
-
-      # This returns all objects after the current timestamp.
-      # By using the 'xxx' suffix, none of the objects of the given timestamp are returned.
-      # S3 ensures alphabetical order of the results, so this works fine for date ranges.
-      bucket.objects(marker: "#{folder}/#{cursor}/xxx", prefix: "#{folder}/")
-    end
-  end
-end