dogtag 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 040cf6345d43641af35bd1982b190cde7143c481
+  data.tar.gz: d2f72279c3599a6fc07bee47683b0a3d431bc425
+SHA512:
+  metadata.gz: a0d87a8746ea3e7c9fb47a49ebb68e326810d871c59046119a319c2847cf149b7f14c2cd9b0f094ff6d5cb9dce39ba8d9ce6cbb3e571db942c50b27823536e39
+  data.tar.gz: ee27ededce69e8de0f1e6c1f859ef05e9c693f98d8616c60d574001843a334946893d9182101a26bb89d6ea6705dd6bc70774aca2d2f8867312c2430a70ba8fc

data/lib/dogtag.rb

@@ -0,0 +1,48 @@
+require 'redis'
+require 'dogtag/mixins/redis'
+
+module Dogtag
+  extend Dogtag::Mixins::Redis
+
+  CUSTOM_EPOCH = 1483228800000 # in milliseconds
+
+  TIMESTAMP_BITS = 41
+  LOGICAL_SHARD_ID_BITS = 10
+  SEQUENCE_BITS = 12
+
+  SEQUENCE_SHIFT = 0
+  LOGICAL_SHARD_ID_SHIFT = SEQUENCE_BITS
+  TIMESTAMP_SHIFT = SEQUENCE_BITS + LOGICAL_SHARD_ID_BITS
+
+  LOGICAL_SHARD_ID_KEY = 'dogtag-generator-logical-shard-id'.freeze
+
+  def self.logical_shard_id=(logical_shard_id)
+    redis.set LOGICAL_SHARD_ID_KEY, logical_shard_id
+  end
+
+  def self.generate_id
+    Generator.new(1).ids.first
+  end
+
+  def self.generate_ids(count)
+    ids = []
+
+    # The Lua script can't always return as many IDs as you may want. So we loop
+    # until we have the exact amount.
+    while ids.length < count
+      initial_id_count = ids.length
+      ids += Generator.new(count - ids.length).ids
+
+      # Ensure the ids array keeps growing as infinite loop insurance
+      return ids unless ids.length > initial_id_count
+    end
+
+    ids
+  end
+end
+
+require 'dogtag/generator'
+require 'dogtag/id'
+require 'dogtag/request'
+require 'dogtag/response'
+require 'dogtag/timestamp'

data/lib/dogtag/generator.rb

@@ -0,0 +1,34 @@
+module Dogtag
+  class Generator
+    def initialize(count = 1)
+      @count = count
+    end
+
+    def ids
+      response.sequence.map do |sequence|
+        (
+          shifted_timestamp |
+          shifted_logical_shard_id |
+          (sequence << Dogtag::SEQUENCE_SHIFT)
+        )
+      end
+    end
+
+    private
+
+    attr_reader :count
+
+    def shifted_timestamp
+      timestamp = Timestamp.from_redis(response.seconds, response.microseconds_part)
+      timestamp.with_epoch(Dogtag::CUSTOM_EPOCH).milliseconds << Dogtag::TIMESTAMP_SHIFT
+    end
+
+    def shifted_logical_shard_id
+      response.logical_shard_id << Dogtag::LOGICAL_SHARD_ID_SHIFT
+    end
+
+    def response
+      @response ||= Request.new(count).response
+    end
+  end
+end

data/lib/dogtag/id.rb

@@ -0,0 +1,29 @@
+module Dogtag
+  class Id
+    SEQUENCE_MAP = ~(-1 << Dogtag::SEQUENCE_BITS) << Dogtag::SEQUENCE_SHIFT
+    LOGICAL_SHARD_ID_MAP = (~(-1 << Dogtag::LOGICAL_SHARD_ID_BITS)) << Dogtag::LOGICAL_SHARD_ID_SHIFT
+    TIMESTAMP_MAP = ~(-1 << Dogtag::TIMESTAMP_BITS) << Dogtag::TIMESTAMP_SHIFT
+
+    attr_reader :id
+
+    def initialize(id)
+      @id = id
+    end
+
+    def custom_timestamp
+      (id & TIMESTAMP_MAP) >> Dogtag::TIMESTAMP_SHIFT
+    end
+
+    def timestamp
+      @timestamp ||= Timestamp.new(custom_timestamp, epoch: Dogtag::CUSTOM_EPOCH)
+    end
+
+    def logical_shard_id
+      (id & LOGICAL_SHARD_ID_MAP) >> Dogtag::LOGICAL_SHARD_ID_SHIFT
+    end
+
+    def sequence
+      (id & SEQUENCE_MAP) >> Dogtag::SEQUENCE_SHIFT
+    end
+  end
+end

data/lib/dogtag/mixins/redis.rb

@@ -0,0 +1,10 @@
+module Dogtag
+  module Mixins
+    module Redis
+      def redis
+        # TODO: Redis config
+        @redis ||= ::Redis.new
+      end
+    end
+  end
+end

data/lib/dogtag/request.rb

@@ -0,0 +1,63 @@
+module Dogtag
+  class Request
+    include Dogtag::Mixins::Redis
+
+    MAX_SEQUENCE = ~(-1 << Dogtag::SEQUENCE_BITS)
+    MIN_LOGICAL_SHARD_ID = 1
+    MAX_LOGICAL_SHARD_ID = ~(-1 << Dogtag::LOGICAL_SHARD_ID_BITS)
+    LUA_SCRIPT_PATH = 'lua/id-generation.lua'.freeze
+    MAX_TRIES = 5
+
+    def initialize(count = 1)
+      raise ArgumentError, 'count must be a number' unless count.is_a? Numeric
+      raise ArgumentError, 'count must be greater than zero' unless count > 0
+
+      @tries = 0
+      @count = count
+    end
+
+    def response
+      Response.new(try_redis_response)
+    end
+
+    private
+
+    attr_reader :count
+
+    def lua_script
+      File.read(
+        File.expand_path "../../#{LUA_SCRIPT_PATH}", File.dirname(__FILE__)
+      )
+    end
+
+    def lua_args
+      lua_args = [
+        MAX_SEQUENCE,
+        MIN_LOGICAL_SHARD_ID,
+        MAX_LOGICAL_SHARD_ID,
+        count
+      ]
+    end
+
+    # NOTE: If too many requests come in inside of a millisecond the Lua script
+    # will lock for 1ms and throw an error. This is meant to retry in those cases.
+    def try_redis_response
+      begin
+        @tries += 1
+        redis_response
+      rescue Redis::CommandError => err
+        if @tries < MAX_TRIES
+          # Exponentially sleep more and more on each try
+          sleep (@tries * @tries).to_f / 900
+          retry
+        else
+          raise err
+        end
+      end
+    end
+
+    def redis_response
+      @redis_response ||= redis.eval(lua_script, keys: lua_args)
+    end
+  end
+end

data/lib/dogtag/response.rb

@@ -0,0 +1,41 @@
+module Dogtag
+  class Response
+    START_SEQUENCE_INDEX = 0
+    END_SEQUENCE_INDEX = 1
+    LOGICAL_SHARD_ID_INDEX = 2
+    SECONDS_INDEX = 3
+    MICROSECONDS_INDEX = 4
+
+    def initialize(redis_response)
+      @redis_response = redis_response
+    end
+
+    def sequence
+      start_sequence..end_sequence
+    end
+
+    def start_sequence
+      redis_response[START_SEQUENCE_INDEX]
+    end
+
+    def end_sequence
+      redis_response[END_SEQUENCE_INDEX]
+    end
+
+    def logical_shard_id
+      redis_response[LOGICAL_SHARD_ID_INDEX]
+    end
+
+    def seconds
+      redis_response[SECONDS_INDEX]
+    end
+
+    def microseconds_part
+      redis_response[MICROSECONDS_INDEX]
+    end
+
+    private
+
+    attr_reader :redis_response
+  end
+end

data/lib/dogtag/timestamp.rb

@@ -0,0 +1,48 @@
+module Dogtag
+  class Timestamp
+    ONE_SECOND_IN_MILLIS = 1_000
+    ONE_MILLI_IN_MICRO_SECS = 1_000
+
+    attr_reader :milliseconds, :epoch
+
+    def initialize(milliseconds, epoch: 0)
+      @milliseconds = milliseconds
+      @epoch = epoch
+    end
+
+    def seconds
+      (milliseconds / ONE_SECOND_IN_MILLIS).floor
+    end
+
+    def microseconds_part
+      (milliseconds - (seconds * ONE_SECOND_IN_MILLIS)) * ONE_MILLI_IN_MICRO_SECS
+    end
+
+    alias to_i milliseconds
+
+    def to_time
+      Time.at(with_unix_epoch.seconds, with_unix_epoch.microseconds_part)
+    end
+
+    def with_unix_epoch
+      @with_unix_epoch ||= with_epoch(0)
+    end
+
+    def with_epoch(new_epoch)
+      new_milliseconds = milliseconds - (new_epoch - epoch)
+
+      self.class.new(new_milliseconds, epoch: new_epoch)
+    end
+
+    def self.from_redis(seconds_part, microseconds_part)
+      # NOTE: we're dropping the microseconds here because we don't need that
+      # level of precision
+      milliseconds = (
+        (seconds_part * ONE_SECOND_IN_MILLIS) +
+        (microseconds_part / ONE_MILLI_IN_MICRO_SECS)
+      )
+
+      new(milliseconds)
+    end
+  end
+end

data/lib/dogtag/version.rb

@@ -0,0 +1,3 @@
+module Dogtag
+  VERSION = '0.1.0'.freeze
+end

data/lua/id-generation.lua

@@ -0,0 +1,61 @@
+local lock_key = 'dogtag-generator-lock'
+local sequence_key = 'dogtag-generator-sequence'
+local logical_shard_id_key = 'dogtag-generator-logical-shard-id'
+
+local max_sequence = tonumber(KEYS[1])
+local min_logical_shard_id = tonumber(KEYS[2])
+local max_logical_shard_id = tonumber(KEYS[3])
+local num_ids = tonumber(KEYS[4])
+
+if redis.call('EXISTS', lock_key) == 1 then
+  redis.log(redis.LOG_NOTICE, 'Dogtag: Cannot generate ID, waiting for lock to expire.')
+  return redis.error_reply('Dogtag: Cannot generate ID, waiting for lock to expire.')
+end
+
+--[[
+Increment by a set number, this can
+--]]
+local end_sequence = redis.call('INCRBY', sequence_key, num_ids)
+local start_sequence = end_sequence - num_ids + 1
+local logical_shard_id = tonumber(redis.call('GET', logical_shard_id_key)) or -1
+
+if end_sequence >= max_sequence then
+  --[[
+  As the sequence is about to roll around, we can't generate another ID until we're sure we're not in the same
+  millisecond since we last rolled. This is because we may have already generated an ID with the same time and
+  sequence, and we cannot allow even the smallest possibility of duplicates. It's also because if we roll the sequence
+  around, we will start generating IDs with smaller values than the ones previously in this millisecond - that would
+  break our k-ordering guarantees!
+
+  The only way we can handle this is to block for a millisecond, as we can't store the time due the purity constraints
+  of Redis Lua scripts.
+
+  In addition to a neat side-effect of handling leap seconds (where milliseconds will last a little bit longer to bring
+  time back to where it should be) because Redis uses system time internally to expire keys, this prevents any duplicate
+  IDs from being generated if the rate of generation is greater than the maximum sequence per millisecond.
+
+  Note that it only blocks even it rolled around *not* in the same millisecond; this is because unless we do this, the
+  IDs won't remain ordered.
+  --]]
+  redis.log(redis.LOG_NOTICE, 'Dogtag: Rolling sequence back to the start, locking for 1ms.')
+  redis.call('SET', sequence_key, '-1')
+  redis.call('PSETEX', lock_key, 1, 'lock')
+  end_sequence = max_sequence
+end
+
+--[[
+The TIME command MUST be called after anything that mutates state, or the Redis server will error the script out.
+This is to ensure the script is "pure" in the sense that randomness or time based input will not change the
+outcome of the writes.
+
+See the "Scripts as pure functions" section at http://redis.io/commands/eval for more information.
+--]]
+local time = redis.call('TIME')
+
+return {
+  start_sequence,
+  end_sequence, -- Doesn't need conversion, the result of INCR or the variable set is always a number.
+  logical_shard_id,
+  tonumber(time[1]),
+  tonumber(time[2])
+}

metadata

@@ -0,0 +1,80 @@
+--- !ruby/object:Gem::Specification
+name: dogtag
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Adam Crownoble
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-05-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.5'
+description: Generate unique IDs with Redis for distributed systems, based heavily
+  off of Icicle and Twitter Snowflake
+email: adam@codenoble.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/dogtag.rb
+- lib/dogtag/generator.rb
+- lib/dogtag/id.rb
+- lib/dogtag/mixins/redis.rb
+- lib/dogtag/request.rb
+- lib/dogtag/response.rb
+- lib/dogtag/timestamp.rb
+- lib/dogtag/version.rb
+- lua/id-generation.lua
+homepage: https://github.com/zillyinc/dogtag
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.11
+signing_key: 
+specification_version: 4
+summary: A Redis-powered Ruby ID generation client
+test_files: []