redis_app_join 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f0254b1ea1caa6281771b11bb2457c7c27e0e1ca
-  data.tar.gz: f51c7597abee8e79fec7034a9fce12f5f1e1782d
+  metadata.gz: eb2846a158c52758eb48351fa5ef8c2fbffdb56f
+  data.tar.gz: 0d1501faf05b6eb7cf0e327e39dd9c4e58347a9d
 SHA512:
-  metadata.gz: bf7863cac7483b98c5bf03b8aa20794d98b7d907a3764e23264ec0e5f578ddc471901a48c045fe5cd3b6121f4b6b71b1af1ce739d3d3398c3eab383861bdc75f
-  data.tar.gz: 35b9feb9220403d25edb0be5402430a15998705ad127c64a6c385562e55fdd62c820ab3e37829cdc6bbff041cd2818ce6bc83af94f204a966f2da0be2fd82e80
+  metadata.gz: a95534d284153f796aa7a93d3697c3aa4285e56b7cdcf2d44e4f9bed0dce5c3b7bd7fd8d393f1f0b253e171cf4816a14d2e451e1e483a3d283bd7ffd7ae68eb9
+  data.tar.gz: 7b38be8d008bf4b543c13bd53913519cc309e163ac2a6e3a321ce5caa0f626aebee5bfef7ba28c6a843fdcd5ecb727a5894ed69181746bb065b60e727c3efa0e

data/.gitignore

@@ -9,3 +9,4 @@
 /tmp/
 
 *.gem
+.byebug_history

data/README.md

@@ -1,6 +1,6 @@
 # RedisAppJoin
 
-Sometimes we need to implement application level joins.  It is easy to query User table and get list of user_ids and then query the child record table for records that belong to those users.  But what if we need to combine data attributes from both tables?  This can also be a use case when querying mutliple databases or 3rd party APIs.  
+Sometimes you need to implement application level joins.  It is easy to query User table and get list of user_ids and then query the child record table for records that belong to those users.  But what if you need to combine data attributes from both tables?  This can also be a use case when querying mutliple databases or 3rd party APIs.  
 
 You can use Redis Hashes as a place to cache data needed as you are looping through records.  Warning - this is ALPHA quality software, be careful before running it in production.  
 
@@ -22,14 +22,14 @@ Or install it yourself as:
 
 ## Usage
 
-Create config/initializers/redis_app_join.rb or place this in environment specific config file.  You can use a different namespace, DB, driver, etc.  
+Create `config/initializers/redis_app_join.rb` or place this in environment specific config file.  You can use a different namespace, DB, driver, etc.  
 
 ```ruby
 redis_conn = Redis.new(host: 'localhost', port: 6379, db: 0)
 REDIS_APP_JOIN = Redis::Namespace.new(:appjoin, redis: redis_conn)
 ```
 
-In the Ruby class where you need to implement application-side join add `include RedisAppJoin`.  Here is a sample report generator that will produce a report of comments created since yesterday and include associated article title and name of user who wrote the article.
+In the Ruby class where you need to implement application-side join add `include RedisAppJoin`.  Here is a sample report generator that will query DB to produce a report of comments created since yesterday and include associated article title and name of user who wrote the article.
 
 ```ruby
 class ReportGen
@@ -39,7 +39,7 @@ class ReportGen
     cache_records(records: comments)
     comment_ids = comments.pluck(:id)
     # =>
-    # => we also could have done  comments.pluck(:article_id)
+    # => you also could have done  comments.pluck(:article_id)
     article_ids = fetch_records_field(record_class: 'Comment', record_ids: comment_ids, field: 'article_id')
     articles = Article.in(id: article_ids).only(:title, :user_id)
     cache_records(records: articles)
@@ -47,7 +47,7 @@ class ReportGen
     user_ids = fetch_records_field(record_class: 'Article', record_ids: article_ids, field: 'user_id')
     users = User.in(id: user_ids).only(:name)
     cache_records(records: users)
-    # => instead of using cached comments we could query DB again
+    # => instead of using cached comments you could query DB again
     cached_comments = fetch_records(record_class: 'Comment', record_ids: comment_ids)
     cached_comments.each do |comment|
       article = fetch_records(record_class: 'Article', record_ids: [comment.article_id]).first
@@ -85,17 +85,41 @@ Comment, Article and User records will be returned like this.
 
 You can do `article.title` and `user = fetch_records(record_class: 'User', record_ids: [article.user_id]).first`.
 
-### TODO:
+### Querying 3rd party APIs
+
+When you query APIs (like [GitHub](https://api.github.com/users/dmitrypol)) you get back JSON.  You might want to correlate this with data from different APIs or internal DBs.  You can cache it in Redis while you are running the process and persist only what you need.  Since these records are not ActiveModels you need to specify the `record_class` which will be part of the Redis key to ensure uniqueness.  
+
+```ruby
+class DataDownloader
+  include RedisAppJoin
+  def perform
+    profiles = User.where(...).only(:profile).pluck(:profile)
+    profiles.each do |p|
+      url = "https://api.github.com/users/#{p}"
+      data = HTTP.get(url).slice(:name, :bio, :location)
+      cache_records(records: data, record_class: 'Github')
+    end
+  end
+end
+# => here is my profile https://api.github.com/users/dmitrypol
+{"db":0,"key":"appjoin:Github:210308","ttl":-1,"type":"hash","value":{"name":"...","bio":"...","location":"..."}}
+```
+
+When you delete such records you need to `delete_records(records: profiles, record_class: 'Github')`.  
 
-Write tests
+### Other config options
 
-Default TTL of 1.week
+If you do not call `delete_records` after you are done all data cached in Redis will expire in 1 week.  Set `REDIS_APP_JOIN_TTL = 1.day` to modify this behavior.  Or set `REDIS_APP_JOIN_TTL = -1` to not expire records.
+
+The gem uses [Redis pipelining](http://redis.io/topics/pipelining) in default batches of 100.  To change that set `REDIS_APP_JOIN_BATCH = 1000` in your initializer.
+
+### TODO:
 
-Support JSON structures in caching (getting data from API), not just ActiveModels
+more tests, integrate with CI tool
 
 Support non-string fields.  For example, if your DB supports array fields you cannot store those attributes in Redis hash values.  
 
-Methods to fetch associated records so we can do `article.user.name` from Redis cache.
+Methods to fetch associated records so you can do `article.user.name` from Redis cache.
 
 ## Development
 

data/lib/redis_app_join.rb

@@ -1,45 +1,66 @@
 require "redis_app_join/version"
+require "redis"
+require "redis-namespace"
+require "readthis"
+#require "active_support/concern"
 
 module RedisAppJoin
 
+  # => default of 1 week
+  REDIS_APP_JOIN_TTL = 60*60*24*7
+  # => default batch size for Redis pipelining when caching records
+  REDIS_APP_JOIN_BATCH = 100
+
   # will loop through records creating keys using combination of class and ID.
-  # can combine different record types (Users and Articles) in the same method call
+  # can combine different record types (Users and Articles) in the same method call unless passing hashes
   # record's attributes will be hash fields
   #
   # @see https://github.com/dmitrypol/redis_app_join
   # @param records [Array] ActiveModels to cache
-  def cache_records(records:)
-    records.each do |record|
-      key = [record.class.name, record.id.to_s].join(':')
-      data = record.attributes.except(:_id, :id)
-      REDIS_APP_JOIN.mapped_hmset(key, data)
+  # @param record_class [String] name of class, used when records are NOT ActiveModel
+  # @raise [RuntimeError] if record is missing ID
+  def cache_records(records:, record_class: nil)
+    records.each_slice(REDIS_APP_JOIN_BATCH) do |batch|
+      REDIS_APP_JOIN.pipelined do
+        batch.each do |record|
+          key = get_key_for_record(record: record, record_class: record_class)
+          if record.is_a?(Hash)
+            data = record
+          else
+            data = record.attributes
+          end
+          REDIS_APP_JOIN.mapped_hmset(key, data.except(:_id, :id))
+          REDIS_APP_JOIN.expire(key, REDIS_APP_JOIN_TTL) unless REDIS_APP_JOIN_TTL == -1
+        end
+      end
     end
   end
 
   # used to delete cached records after the process is done
-  # can combine different record types (Users and Articles) in the same method call
+  # can combine different record types (Users and Articles) in the same method call unless passing hashes
   #
   # @param records [Array] ActiveModels to delete
-  def delete_records(records:)
+  # @param record_class [String] name of class, used when records are NOT ActiveModel
+  def delete_records(records:, record_class: nil)
     records.each do |record|
-      key = [record.class.name, record.id.to_s].join(':')
+      key = get_key_for_record(record: record, record_class: record_class)
       REDIS_APP_JOIN.del(key)
     end
   end
 
-  # fetch recors from cache,
+  # fetch records from cache,
   # cannot combine different record types (Users and Articles) in the same method call
   #
-  # @param record_class [String] - name of class, used in lookup
+  # @param record_class [String] name of class, used in lookup
   # @param record_ids [Array] array of IDs to lookup
   # @return [Array] array of objects and include the original record ID as one of the attributes for each object.
   def fetch_records(record_class:, record_ids:)
     output = []
     record_ids.each do |record_id|
-      key = [record_class, record_id.to_s].join(':')
+      key = get_key_for_record_id(record_id: record_id, record_class: record_class)
       data = REDIS_APP_JOIN.hgetall(key)
-      # => add the key as ID attribute
-      output << OpenStruct.new(data.merge(id: record_id.to_s))
+      # => add the key as ID attribute if there is data hash
+      output << OpenStruct.new(data.merge(id: record_id.to_s)) if data.size > 0
     end
     return output
   end
@@ -48,18 +69,48 @@ module RedisAppJoin
   # only returns the field if it's present
   # cannot combine different record types in the same method call
   #
-  # @param record_class [String] - name of class, used in lookup
+  # @param record_class [String] name of class, used in lookup
   # @param record_ids [Array] array of IDs to lookup
   # @param field [String] name of field/attribute to retrieve
   # @return [Array] array of unique strings
   def fetch_records_field(record_class:, record_ids:, field:)
     output = []
     record_ids.each do |record_id|
-      key = [record_class, record_id.to_s].join(':')
+      key = get_key_for_record_id(record_id: record_id, record_class: record_class)
       data = REDIS_APP_JOIN.hget(key, field)
-      output << data
+      output << data if data # checks if nil
     end
     return output.uniq
   end
 
+private
+
+  # creates a key for specific record id and class
+  #
+  # @param record [Object]
+  # @param record_class [String]
+  # @return [String]
+  def get_key_for_record (record:, record_class:)
+    if record.is_a?(Hash)
+      record_id = record[:id] || record[:_id]
+    else
+      record_id = record.id
+    end
+    raise RuntimeError, 'missing record_id' if record_id == nil
+    record_class ||= record.class.name
+    raise RuntimeError, 'missing record_class' if ['', nil, 'Hash'].include?(record_class)
+    key = [record_class, record_id.to_s].join(':')
+    return key
+  end
+
+  # @param record_id [String]
+  # @param record_class [String]
+  # @return [String]
+  def get_key_for_record_id (record_id:, record_class:)
+    raise RuntimeError, 'missing record_id' if record_id == nil
+    raise RuntimeError, 'missing record_class' if ['', nil, 'Hash'].include?(record_class)
+    key = [record_class, record_id.to_s].join(':')
+    return key
+  end
+
 end

data/lib/redis_app_join/version.rb

@@ -1,3 +1,3 @@
 module RedisAppJoin
-  VERSION = "0.1.1"
+  VERSION = "0.1.2"
 end

data/redis_app_join.gemspec

@@ -32,7 +32,10 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rspec", "~> 3.0"
   # =>
   spec.add_development_dependency 'mock_redis', '~> 0.17', '>= 0.17.0'
+  spec.add_development_dependency 'rspec-activemodel-mocks', '>=1.0.3'
+  spec.add_development_dependency 'byebug', '>=9.0.6'
   # =>
+  #spec.add_runtime_dependency 'activesupport', '~> 4.2', '>= 4.2.7'
   spec.add_runtime_dependency 'redis', '~> 3.3'
   spec.add_runtime_dependency 'redis-namespace', '~> 1.5'
   spec.add_runtime_dependency 'readthis', '~> 1.3', '>= 1.3.0'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: redis_app_join
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Dmitry Polyakovsky
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-10-19 00:00:00.000000000 Z
+date: 2016-10-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -72,6 +72,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.17.0
+- !ruby/object:Gem::Dependency
+  name: rspec-activemodel-mocks
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.3
+- !ruby/object:Gem::Dependency
+  name: byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 9.0.6
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 9.0.6
 - !ruby/object:Gem::Dependency
   name: redis
   requirement: !ruby/object:Gem::Requirement