fmcache 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 5937e1aa882967a35ee510c8ca9dda6abe03cd4b0a493aad50d75526a8dc24c4
+  data.tar.gz: 25b66cbedff899c9e0565686e709fd6dc4211c2f3f42384fbfaa2e42607b9946
+SHA512:
+  metadata.gz: fbc1e97480a2d4ce1dfe33dbf2b5b6e35560281309e4a4fdb927c458177287e5202cd12196420ed0fda551c491faee6ff38207358f715a067fde5347d3e3c7c3
+  data.tar.gz: f8d1edeb81200ca8418d91356b072a5dfa0f3ee8edc63df5381ef986c2466a6cc14e883631e976d325b2ed28492e9081b8902184bdc12dcfcff98094d5a20189

data/.gitignore

@@ -0,0 +1,8 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.rspec

@@ -0,0 +1 @@
+--require spec_helper

data/.travis.yml

@@ -0,0 +1,8 @@
+sudo: false
+language: ruby
+rvm:
+- 2.5.1
+before_install:
+- gem install bundler -v 1.16.2
+script:
+- bundle exec rspec

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in fmcache.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,67 @@
+PATH
+  remote: .
+  specs:
+    fmcache (0.1.0)
+      field_mask_parser (~> 0.4.1)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activemodel (5.2.2)
+      activesupport (= 5.2.2)
+    activerecord (5.2.2)
+      activemodel (= 5.2.2)
+      activesupport (= 5.2.2)
+      arel (>= 9.0)
+    activesupport (5.2.2)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 0.7, < 2)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
+    arel (9.0.0)
+    coderay (1.1.2)
+    concurrent-ruby (1.1.3)
+    diff-lcs (1.3)
+    field_mask_parser (0.4.1)
+    i18n (1.2.0)
+      concurrent-ruby (~> 1.0)
+    method_source (0.9.2)
+    minitest (5.11.3)
+    mock_redis (0.19.0)
+    pry (0.12.2)
+      coderay (~> 1.1.0)
+      method_source (~> 0.9.0)
+    rake (10.5.0)
+    redis (4.0.3)
+    rspec (3.8.0)
+      rspec-core (~> 3.8.0)
+      rspec-expectations (~> 3.8.0)
+      rspec-mocks (~> 3.8.0)
+    rspec-core (3.8.0)
+      rspec-support (~> 3.8.0)
+    rspec-expectations (3.8.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-mocks (3.8.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-support (3.8.0)
+    thread_safe (0.3.6)
+    tzinfo (1.2.5)
+      thread_safe (~> 0.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  activerecord (~> 5.2)
+  bundler (~> 1.16)
+  fmcache!
+  mock_redis (~> 0.19)
+  pry (~> 0.11)
+  rake (~> 10.0)
+  redis (~> 4.0)
+  rspec (~> 3.8)
+
+BUNDLED WITH
+   1.16.2

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2018 Nao Minami
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,61 @@
+# FMCache
+
+Library for caching json masked by FieldMask
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'fmcache'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install fmcache
+
+## Usage
+
+You can get data from the cache by using `FMCache::Engine#fetch`. If there is uncached data, you can pass the block and fetch the rest from the other data source.
+
+An example of code is shown below.
+
+```ruby
+[4] pry(main)> redis = Redis.new(url: "redis://localhost:6379")
+=> #<Redis client v4.0.3 for redis://localhost:6379>
+
+[5] pry(main)> cache_engine = FMCache::Engine.new(client: redis, fm_parser: -> (fields) { FieldMaskParser.parse(paths: fields, root: User) })
+=> #<FMCache::Engine:0x00007fb5f8f985e8
+ @client=#<FMCache::Client:0x00007fb5f8f98598 @client=#<Redis client v4.0.3 for redis://localhost:6379>, @notifier=nil>,
+ @decoder=
+  #<FMCache::Decoder:0x00007fb5f8f984d0
+   @field_mask_parser=#<Proc:XXX (lambda)>,
+   @fields_checker=#<FMCache::Decoder::FieldsChecker:0x00007fb5f8f98458>,
+   @value_decoder=#<FMCache::Decoder::ValueDecoder:0x00007fb5f8f98480>>,
+ @encoder=#<FMCache::Encoder:0x00007fb5f8f984f8>,
+ @fm_parser=#<Proc:XXX (lambda)>,
+ @ttl=604800>
+
+[7] pry(main)> cache_engine.fetch(ids: [1], field_mask: FieldMaskParser.parse(paths: ["id", "name"], root: User)) do |ids, field_mask|
+[7] pry(main)*   fetch_json(ids, field_mask)
+[7] pry(main)* end
+=> [{:id=>1, :name=>"Taro"}]
+```
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake true` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/south37/fmcache.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+task :default => :spec

data/bin/console

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "fmcache"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+require "pry"
+Pry.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/fmcache.gemspec

@@ -0,0 +1,34 @@
+
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "fmcache/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "fmcache"
+  spec.version       = FMCache::VERSION
+  spec.authors       = ["Nao Minami"]
+  spec.email         = ["south37777@gmail.com"]
+
+  spec.summary       = %q{Library for caching json masked by FieldMask}
+  spec.description   = %q{Library for caching json masked by FieldMask}
+  spec.homepage      = "https://github.com/south37/fmcache"
+  spec.license       = "MIT"
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.16"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.8"
+  spec.add_development_dependency "pry", "~> 0.11"
+  spec.add_development_dependency "mock_redis", "~> 0.19"
+  spec.add_development_dependency "redis", "~> 4.0"
+  spec.add_development_dependency "activerecord", "~> 5.2"
+  spec.add_dependency "field_mask_parser", "~> 0.4.1"
+end

data/lib/fmcache/client.rb

@@ -0,0 +1,81 @@
+module FMCache
+  class Client
+    # @param [Redis | MockRedis] client
+    # @param [Proc] notifier
+    def initialize(client, notifier = nil)
+      @client   = client
+      @notifier = notifier
+    end
+
+    attr_reader :client, :notifier
+
+    # @param [{ String => { String => String } }] values
+    # @return [Boolean]
+    def set(values:, ttl:)
+      client.pipelined do
+        values.each do |h_key, h_values|
+          client.mapped_hmset(h_key, h_values)
+          client.expire(h_key, ttl)
+        end
+      end
+
+      true
+    rescue Redis::BaseConnectionError => e
+      notify(e)
+      false
+    end
+
+    # @param [<String>] keys
+    # @param [<String>] fields
+    # @return [{ String => { String => String } }]
+    def get(keys:, fields:)
+      return {} if keys.size == 0
+
+      values = client.pipelined do
+        keys.each do |key|
+          client.mapped_hmget(key, *fields)
+        end
+      end
+      keys.zip(values).to_h
+    rescue Redis::BaseConnectionError => e
+      notify(e)
+      keys.map { |k| [k, fields.map { |f| [f, nil] }.to_h] }.to_h
+    end
+
+    # @param [<String>] keys
+    # @return [Boolean]
+    def del(keys:)
+      if keys.size > 0
+        client.del(*keys)
+      end
+      true
+    rescue Redis::BaseConnectionError => e
+      notify(e)
+      false
+    end
+
+    # @param [<String>] keys
+    # @param [<String>] fields
+    # @return [Boolean]
+    def hdel(keys:, fields:)
+      client.pipelined do
+        keys.each do |key|
+          fields.each do |field|
+            client.hdel(key, field)
+          end
+        end
+      end
+      true
+    rescue Redis::BaseConnectionError => e
+      notify(e)
+      false
+    end
+
+  private
+
+    # @param [Exception] e
+    def notify(e)
+      notifier.call(e) if notifier
+    end
+  end
+end

data/lib/fmcache/decoder/decode_result.rb

@@ -0,0 +1,13 @@
+module FMCache
+  class Decoder
+    class DecodeResult
+      def initialize(values:, invalid_values:, invalid_fields:)
+        @values         = values
+        @invalid_values = invalid_values
+        @invalid_fields = invalid_fields
+      end
+
+      attr_reader :values, :invalid_values, :invalid_fields
+    end
+  end
+end

data/lib/fmcache/decoder/fields_checker.rb

@@ -0,0 +1,78 @@
+module FMCache
+  class Decoder
+    class FieldsChecker
+      class CheckResult
+        def initialize(list:, incomplete_list:, missing_fields:)
+          @list            = list
+          @incomplete_list = incomplete_list
+          @missing_fields  = missing_fields
+        end
+
+        attr_reader :list, :incomplete_list, :missing_fields
+      end
+
+      # @param [<Hash>] list
+      # @param [FieldMaskParser::Node] field_mask
+      # @return [CheckResult]
+      def check(list, field_mask)
+        l              = []
+        incomplete_l   = []
+        missing_fields = Set.new
+
+        list.each do |d|
+          _, m_fields = check_fields(d, field_mask)
+          if m_fields.size == 0
+            l << d
+          else
+            incomplete_l << d
+            missing_fields |= m_fields
+          end
+        end
+
+        CheckResult.new(
+          list:            l,
+          incomplete_list: incomplete_l,
+          missing_fields:  missing_fields,
+        )
+      end
+
+      # @param [Hash] data
+      # @param [FieldMaskParser::Node] field_mask
+      def check_fields(data, field_mask, prefix = [])
+        fields         = []
+        missing_fields = []
+
+        id_exists = false
+        field_mask.attrs.each do |attr|
+          f = Field.to_s(prefix: prefix, attr: attr)
+          if data.fetch(f)
+            fields << f
+            id_exists = true if attr == :id
+          else  # NOTE: When nil, cache of the field does not exist
+            missing_fields << f
+          end
+        end
+
+        if id_exists
+          field_mask.assocs.each do |assoc|
+            f, m_f = check_fields(data, assoc, (prefix + [assoc.name]))
+            fields         += f
+            missing_fields += m_f
+          end
+        else
+          # NOTE: When the cach of id does not exist, treat current and lower
+          # layer as missing fields
+          # TODO(south37) Improve performance
+          missing_fields += fields
+          fields         = []
+
+          field_mask.assocs.each do |assoc|
+            missing_fields += Helper.to_fields(assoc, prefix: prefix + [assoc.name])
+          end
+        end
+
+        [fields, missing_fields]
+      end
+    end
+  end
+end

data/lib/fmcache/decoder/value_decoder/data.rb

@@ -0,0 +1,66 @@
+module FMCache
+  class Decoder
+    class ValueDecoder
+      class Data
+        def initialize(field_mask:)
+          has_one_names  = field_mask.has_ones.map(&:name)
+          has_many_names = field_mask.has_manies.map(&:name)
+
+          @attr_names = Set.new(field_mask.attrs)
+          @attrs      = {}
+          @has_ones   = has_one_names.map { |n| [n, nil] }.to_h
+          @has_manies = has_many_names.map { |n| [n, []] }.to_h
+        end
+
+        def push_attr(name:, item:)
+          if !@attr_names.include?(name)
+            raise "invalid data"
+          end
+          @attrs[name] = item
+        end
+
+        def push_has_one(name:, data:)
+          if !@has_ones.has_key?(name)
+            raise "invalid data"
+          end
+          @has_ones[name] = data
+        end
+
+        def push_has_many(name:, data:)
+          if !@has_manies.has_key?(name)
+            raise "inavlid data"
+          end
+          @has_manies[name] << data
+        end
+
+        def valid?
+          @attr_names == Set.new(@attrs.keys)
+        end
+
+        def id
+          raise "internal error!" if @attrs.size == 0
+          @attrs.values[0].id
+        end
+
+        def p_id
+          raise "internal error!" if @attrs.size == 0
+          @attrs.values[0].p_id
+        end
+
+        def to_h
+          r = { id: id }
+          @attrs.each do |name, item|
+            r[name] = item.value
+          end
+          @has_ones.each do |name, data|
+            r[name] = data.to_h
+          end
+          @has_manies.each do |name, data_list|
+            r[name] = data_list.map(&:to_h)
+          end
+          r
+        end
+      end
+    end
+  end
+end

data/lib/fmcache/decoder/value_decoder/item.rb

@@ -0,0 +1,15 @@
+module FMCache
+  class Decoder
+    class ValueDecoder
+      class Item
+        def initialize(id:, p_id:, value:)
+          @id    = id
+          @p_id  = p_id
+          @value = value
+        end
+
+        attr_reader :id, :p_id, :value
+      end
+    end
+  end
+end

data/lib/fmcache/decoder/value_decoder.rb

@@ -0,0 +1,113 @@
+require "fmcache/decoder/value_decoder/data"
+require "fmcache/decoder/value_decoder/item"
+
+module FMCache
+  class Decoder
+    class ValueDecoder
+      # @param [Hash] data
+      # @param [FieldMaskParser::Node] field_mask
+      # @return [<Hash, <String>>]
+      def decode(data, field_mask)
+        @data           = data
+        @invalid_fields = Set.new
+
+        h = fetch(field_mask, [])
+
+        d = h.values.first  # NOTE: Top of h is a user
+
+        [d.to_h, @invalid_fields]
+      end
+
+    private
+
+      # @param [FieldMaskParser::Node] field_mask
+      # @param [<Symbol>] prefix
+      # @return [{ Integer => Data }]
+      def fetch(field_mask, prefix)
+        r = fetch_layer(field_mask, prefix)
+        assign_has_one!(r, field_mask, prefix)
+        assign_has_many!(r, field_mask, prefix)
+        r
+      end
+
+      # @param [FieldMaskParser::Node] field_mask
+      # @param [<Symbol>] prefix
+      # @return [{ Integer => Data }]
+      def fetch_layer(field_mask, prefix)
+        r = {}
+
+        fetch_items(field_mask, prefix).each do |attr, items|
+          items.each do |item|
+            r[item.id] ||= Data.new(field_mask: field_mask)
+            r[item.id].push_attr(name: attr, item: item)
+          end
+        end
+
+        r.each do |_, data|
+          if !data.valid?
+            # NOTE: If data is invalid, we treat this layer as invalid.
+            @invalid_fields |= Set.new(Helper.to_fields(field_mask, prefix: prefix))
+          end
+        end
+
+        r
+      end
+
+      def fetch_items(field_mask, prefix)
+        r = {}
+        field_mask.attrs.each do |attr|
+          f = Field.to_s(prefix: prefix, attr: attr)
+          h = @data.fetch(f)
+          if h.nil?
+            raise "invalid json: `#{h}` with field: #{f}"
+          end
+          r[attr] = itemize(h)
+        end
+        r
+      end
+
+      # @param [String] h
+      def itemize(h)
+        h.map do |hh|
+          Item.new(id: hh.fetch(:id), p_id: hh.fetch(:p_id), value: hh.fetch(:value))
+        end
+      end
+
+      # @param [{ Integer => Data }]
+      # @param [FieldMaskParser::Node] field_mask
+      # @param [<Symbol>] prefix
+      def assign_has_one!(parents, field_mask, prefix)
+        field_mask.has_ones.each do |assoc|
+          fetch(assoc, prefix + [assoc.name]).each do |_, data|
+            p = parents[data.p_id]
+            # NOTE: if p is nil, parent layer is inconsistent with this layer.
+            # So we treat them as invalid.
+            if p.nil?
+              @invalid_fields |= Set.new(Helper.to_fields(field_mask, prefix: prefix))
+            else
+              p.push_has_one(name: assoc.name, data: data)
+            end
+          end
+        end
+      end
+
+      # @param [{ Integer => Data }]
+      # @param [FieldMaskParser::Node] field_mask
+      # @param [<Symbol>] prefix
+      def assign_has_many!(parents, field_mask, prefix)
+        field_mask.has_manies.each do |assoc|
+          fetch(assoc, prefix + [assoc.name]).each do |_, data|
+            p = parents[data.p_id]
+            # NOTE: if p is nil, parent layer is inconsistent with this layer.
+            # So we treat them as invalid.
+            if p.nil?
+              @invalid_fields |= Set.new(Helper.to_fields(field_mask, prefix: prefix))
+            else
+              p.push_has_many(name: assoc.name, data: data)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fmcache/decoder.rb

@@ -0,0 +1,86 @@
+require "fmcache/decoder/decode_result"
+require "fmcache/decoder/fields_checker"
+require "fmcache/decoder/value_decoder"
+
+module FMCache
+  class Decoder
+    # @param [Proc] field_mask_parser
+    def initialize(field_mask_parser)
+      @field_mask_parser = field_mask_parser
+      @value_decoder     = ValueDecoder.new
+      @fields_checker    = FieldsChecker.new
+    end
+
+    attr_reader :field_mask_parser, :value_decoder, :fields_checker
+
+    # @param [{ String => { String => <Hash> } }] hash
+    # @param [FieldMaskParser::Node] field_mask
+    # @return [<Hash>, <Hash>, IncompleteInfo]
+    def decode(hash, field_mask)
+      list = hash.values
+      check_result = fields_checker.check(list, field_mask)
+
+      decode_result = decode_list(check_result.list, field_mask: field_mask)
+
+      f = Helper.to_fields(field_mask) - check_result.missing_fields.to_a
+      i_decode_result = decode_list(check_result.incomplete_list, fields: f)
+
+      concat(check_result, decode_result, i_decode_result)
+    end
+
+  private
+
+    # @param [<Hash>] list
+    # @param [FieldMaskParser::Node] field_mask
+    def decode_list(list, field_mask: nil, fields: nil)
+      if field_mask.nil? && fields.nil?
+        raise "invalid args!"
+      end
+      if fields
+        field_mask = field_mask_parser.call(fields.map(&:to_s))
+      end
+
+      values         = []
+      invalid_values = []
+      invalid_fields = Set.new
+
+      list.each do |d|
+        v, i_fields = value_decoder.decode(d, field_mask)
+        if i_fields.size == 0
+          values << v
+        else
+          invalid_values << v
+          invalid_fields |= i_fields
+        end
+      end
+
+      DecodeResult.new(
+        values:         values,
+        invalid_values: invalid_values,
+        invalid_fields: invalid_fields,
+      )
+    end
+
+    # @param [DecodeResult] decode_result
+    # @param [DecodeResult] invalid_decode_result
+    # @return [<Hash>, <Hash>, IncompleteInfo]
+    def concat(check_result, decode_result, invalid_decode_result)
+      v = decode_result.values
+      i_v = decode_result.invalid_values +
+        invalid_decode_result.values +
+        invalid_decode_result.invalid_values
+
+      missing_fields = Set.new(
+        check_result.missing_fields +
+        decode_result.invalid_fields +
+        invalid_decode_result.invalid_fields
+      )
+      incomplete_info = IncompleteInfo.new(
+        ids:        i_v.map { |h| h.fetch(:id) },
+        field_mask: field_mask_parser.call(missing_fields),
+      )
+
+      [v, i_v, incomplete_info]
+    end
+  end
+end

data/lib/fmcache/encoder/itemizer.rb

@@ -0,0 +1,49 @@
+module FMCache
+  class Encoder
+    class Itemizer
+      # @param [Hash] value
+      # @param [FieldMaskParser::Node] field_mask
+      def initialize(value, field_mask)
+        @value      = value
+        @field_mask = field_mask
+        @items      = {}
+      end
+
+      # [{ String => <Hash> }]
+      attr_reader :items
+
+      def run!
+        traverse!(value: @value, field_mask: @field_mask, prefix: [], p_id: nil)
+      end
+
+    private
+
+      def traverse!(value:, field_mask:, prefix:, p_id:)
+        id = value[:id]
+
+        field_mask.attrs.each do |attr|
+          f = Field.to_s(prefix: prefix, attr: attr)
+          v = value[attr]
+          @items[f] ||= []
+          @items[f] << { value: v, id: id, p_id: p_id }
+        end
+
+        field_mask.has_ones.each do |assoc|
+          v      = value[assoc.name] || {}
+          prefix = prefix + [assoc.name]
+
+          traverse!(value: v, field_mask: assoc, prefix: prefix, p_id: id)
+        end
+
+        field_mask.has_manies.each do |assoc|
+          values = value[assoc.name] || []
+          prefix = prefix + [assoc.name]
+
+          values.each do |v|
+            traverse!(value: v, field_mask: assoc, prefix: prefix, p_id: id)
+           end
+        end
+      end
+    end
+  end
+end

data/lib/fmcache/encoder.rb

@@ -0,0 +1,38 @@
+require "fmcache/encoder/itemizer"
+
+module FMCache
+  class Encoder
+    # @param [<Hash>] values
+    # @param [FieldMaskParser::Node] field_mask
+    # @return [{ String => { String => <Hash> } }]
+    def encode(values, field_mask)
+      r = {}
+      values.each do |value|
+        h = {}
+
+        # NOTE: initialize each field by array
+        fields = Helper.to_fields(field_mask).map(&:to_s)
+        fields.each do |f|
+          h[f] = []
+        end
+
+        encode_one(value, field_mask).each do |f, v|
+          h[f] = v
+        end
+
+        id = value.fetch(:id)
+        r[Helper.to_key(id)] = h
+      end
+      r
+    end
+
+    # @param [Hash] value
+    # @param [FieldMaskParser::Node] field_mask
+    # @return [{ String => <Hash> }]
+    def encode_one(value, field_mask)
+      itemizer = Itemizer.new(value, field_mask)
+      itemizer.run!
+      itemizer.items
+    end
+  end
+end

data/lib/fmcache/engine.rb

@@ -0,0 +1,118 @@
+module FMCache
+  class Engine
+    DEFAULT_TTL = 7 * 24 * 3600  # 7 days
+
+    # @param [Redis | MockRRedis] client
+    def initialize(client:, fm_parser:, ttl: DEFAULT_TTL, notifier: nil)
+      @client    = Client.new(client, notifier)
+      @fm_parser = wrap(fm_parser)
+      @ttl       = ttl
+      @encoder   = Encoder.new
+      @decoder   = Decoder.new(@fm_parser)
+    end
+
+    attr_reader :client, :fm_parser, :encoder, :decoder
+
+    # @param [<Hash>] values
+    # @param [FieldMaskParser::Node] field_mask
+    # @return [Boolean]
+    def write(values:, field_mask:)
+      normalize!(field_mask)
+      h = encode(values, field_mask)
+      client.set(values: Helper.jsonize(h), ttl: @ttl)
+    end
+
+    # @param [<Integer | String>] ids
+    # @param [FieldMaskParser::Node] field_mask
+    # @return [<Hash>, <Hash>, IncompleteInfo]
+    def read(ids:, field_mask:)
+      ids = ids.map(&:to_i)
+      normalize!(field_mask)
+
+      keys   = Helper.to_keys(ids)
+      fields = Helper.to_fields(field_mask).map(&:to_s)
+      h = client.get(keys: keys, fields: fields)
+      decode(merge(Helper.dejsonize(h), ids), field_mask)
+    end
+
+    # @param [<Integer | String>] ids
+    # @param [FieldMaskParser::Node] field_mask
+    # @yieldparam [<Integer>, FieldMaskParser::Node] ids, field_mask
+    # @yieldreturn [<Hash>]
+    # @return [<Hash>]
+    def fetch(ids:, field_mask:, &block)
+      ids = ids.map(&:to_i)
+      normalize!(field_mask)
+
+      values, incomplete_values, incomplete_info = read(ids: ids, field_mask: field_mask)
+      return values if incomplete_values.size == 0
+
+      # NOTE: get new data
+      d = block.call(incomplete_info.ids, incomplete_info.field_mask)
+      write(values: d, field_mask: incomplete_info.field_mask)
+
+      older = encode(incomplete_values, field_mask)
+      newer = encode(d,                 incomplete_info.field_mask)
+
+      v, i_v, i_i = decode(older.deep_merge(newer), field_mask)
+
+      # NOTE: Delete invalid data as read repair
+      client.hdel(
+        keys:   Helper.to_keys(i_i.ids),
+        fields: Helper.to_fields(i_i.field_mask),
+      )
+
+      Helper.sort(values + v + i_v, ids)
+    end
+
+    def delete(ids:)
+      ids = ids.map(&:to_i)
+      client.del(keys: Helper.to_keys(ids))
+    end
+
+  private
+
+    # @param [{ String => { String => <Hash> } }] hash
+    # @param [FieldMaskParser::Node] field_mask
+    # @return [<Hash>, <Hash>, IncompleteInfo]
+    def decode(hash, field_mask)
+      decoder.decode(hash, field_mask)
+    end
+
+    # @param [<Hash>] values
+    # @param [FieldMaskParser::Node] field_mask
+    # @rerturn [{ String => { String => <Hash> } }]
+    def encode(values, field_mask)
+      encoder.encode(values, field_mask)
+    end
+
+    # @param [{ String => { String => <Hash> } }] hash
+    # @param [<Integer>] ids
+    # @return [{ String => { String => <Hash> } }]
+    def merge(hash, ids)
+      # NOTE: Set `id` to list. json format must be consistent with Encoder and Decoder
+      ids.each do |id|
+        h = hash.fetch(Helper.to_key(id))
+        h.merge!({ "id" => [{ id: id, p_id: nil, value: id }] })
+      end
+      hash
+    end
+
+    def wrap(fm_parser)
+      -> (fields) {
+        n = fm_parser.call(fields)
+        normalize!(n)
+        n
+      }
+    end
+
+    def normalize!(field_mask)
+      if !field_mask.attrs.include?(:id)
+        field_mask.attrs << :id
+      end
+      field_mask.assocs.each do |a|
+        normalize!(a)
+      end
+    end
+  end
+end

data/lib/fmcache/field.rb

@@ -0,0 +1,10 @@
+module FMCache
+  class Field
+    class << self
+      def to_s(prefix:, attr:)
+        l = prefix + [attr]
+        l.join(".")
+      end
+    end
+  end
+end

data/lib/fmcache/helper.rb

@@ -0,0 +1,76 @@
+module FMCache
+  class Helper
+    class << self
+      # @param [FieldMaskParser::Node] field_mask
+      # @return [<String>]
+      def to_fields(field_mask, prefix: [])
+        r = []
+        field_mask.attrs.each do |attr|
+          r << Field.to_s(prefix: prefix, attr: attr)
+        end
+        field_mask.assocs.each do |assoc|
+          r += to_fields(assoc, prefix: prefix + [assoc.name])
+        end
+        r
+      end
+
+      # @param [<Integer>] ids
+      # @return [<String>]
+      def to_keys(ids)
+        ids.map { |id| to_key(id) }
+      end
+
+      # @param [Integer] id
+      # @return [String]
+      def to_key(id)
+        "fmcache:#{id}"
+      end
+
+      # @param [{ String => { String => <Hash> } }] hash
+      # @return [{ String => { String => String } }]
+      def jsonize(hash)
+        r = {}
+        hash.each do |k, v|
+          h = {}
+          v.each do |kk, vv|
+            h[kk] = vv.to_json
+          end
+          r[k] = h
+        end
+        r
+      end
+
+      # @param [{ String => { String => String } }] hash
+      # @return [{ String => { String => <Hash> } }]
+      def dejsonize(hash)
+        r = {}
+        hash.each do |k, v|
+          h = {}
+          v.each do |kk, vv|
+            if vv.nil?
+              h[kk] = nil
+            else
+              begin
+                h[kk] = JSON.parse(vv, symbolize_names: true)
+              rescue
+                h[kk] = nil
+              end
+            end
+          end
+          r[k] = h
+        end
+        r
+      end
+
+      # @param [<Hash>] values
+      # @param [<Integer>] ids
+      # @return [<Hash>]
+      def sort(values, ids)
+        id_map = ids.map.with_index { |id, i| [id, i] }.to_h
+        values.sort do |a, b|
+          id_map.fetch(a.fetch(:id)) <=> id_map.fetch(b.fetch(:id))
+        end
+      end
+    end
+  end
+end

data/lib/fmcache/incomplete_info.rb

@@ -0,0 +1,23 @@
+module FMCache
+  class IncompleteInfo
+    def initialize(ids:, field_mask:)
+      @ids        = ids
+      @field_mask = field_mask
+    end
+
+    attr_reader :ids, :field_mask
+
+    def ==(other)
+      @ids == other.ids &&
+        @field_mask.to_h == other.field_mask.to_h
+    end
+
+    def eql?(other)
+      self == other
+    end
+
+    def hash
+      @ids.hash ^ @field_mask.hash
+    end
+  end
+end

data/lib/fmcache/version.rb

@@ -0,0 +1,3 @@
+module FMCache
+  VERSION = "0.1.0"
+end

data/lib/fmcache.rb

@@ -0,0 +1,11 @@
+require "fmcache/client"
+require "fmcache/decoder"
+require "fmcache/encoder"
+require "fmcache/engine"
+require "fmcache/field"
+require "fmcache/helper"
+require "fmcache/incomplete_info"
+require "fmcache/version"
+
+module FMCache
+end

metadata

@@ -0,0 +1,182 @@
+--- !ruby/object:Gem::Specification
+name: fmcache
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Nao Minami
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2018-12-12 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.8'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.8'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.11'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.11'
+- !ruby/object:Gem::Dependency
+  name: mock_redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.19'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.19'
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.2'
+- !ruby/object:Gem::Dependency
+  name: field_mask_parser
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.4.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.4.1
+description: Library for caching json masked by FieldMask
+email:
+- south37777@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- fmcache.gemspec
+- lib/fmcache.rb
+- lib/fmcache/client.rb
+- lib/fmcache/decoder.rb
+- lib/fmcache/decoder/decode_result.rb
+- lib/fmcache/decoder/fields_checker.rb
+- lib/fmcache/decoder/value_decoder.rb
+- lib/fmcache/decoder/value_decoder/data.rb
+- lib/fmcache/decoder/value_decoder/item.rb
+- lib/fmcache/encoder.rb
+- lib/fmcache/encoder/itemizer.rb
+- lib/fmcache/engine.rb
+- lib/fmcache/field.rb
+- lib/fmcache/helper.rb
+- lib/fmcache/incomplete_info.rb
+- lib/fmcache/version.rb
+homepage: https://github.com/south37/fmcache
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.6
+signing_key: 
+specification_version: 4
+summary: Library for caching json masked by FieldMask
+test_files: []