dogtag 0.1.0 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 040cf6345d43641af35bd1982b190cde7143c481
-  data.tar.gz: d2f72279c3599a6fc07bee47683b0a3d431bc425
+  metadata.gz: 3873a444c41218413a933994fd93c9683588e6f5
+  data.tar.gz: 561a47c6e821efc9d602d04b985ae1e05e1f1809
 SHA512:
-  metadata.gz: a0d87a8746ea3e7c9fb47a49ebb68e326810d871c59046119a319c2847cf149b7f14c2cd9b0f094ff6d5cb9dce39ba8d9ce6cbb3e571db942c50b27823536e39
-  data.tar.gz: ee27ededce69e8de0f1e6c1f859ef05e9c693f98d8616c60d574001843a334946893d9182101a26bb89d6ea6705dd6bc70774aca2d2f8867312c2430a70ba8fc
+  metadata.gz: 0ddf7a7f873c0c3b6a31c80b189cde967aea42569afbec71aeb95ae3361049176d43a525d4270da2ad4dc701fdab2f64b2f9c9aa69da67cbfbeda5f10a79e9db
+  data.tar.gz: dd72cbd167d3c91f5fd530108a17f1ab4ecf6fe50fd4efca81aaee763bbd5c69a8904d98ab947f2dd7e070a12b828b9bedbec78e543330013a7e544bf319cdb8

data/lib/dogtag.rb

@@ -6,32 +6,58 @@ module Dogtag
 
   CUSTOM_EPOCH = 1483228800000 # in milliseconds
 
-  TIMESTAMP_BITS = 41
-  LOGICAL_SHARD_ID_BITS = 10
-  SEQUENCE_BITS = 12
+  TIMESTAMP_BITS = 40
+  LOGICAL_SHARD_ID_BITS = 5
+  DATA_TYPE_BITS = 8
+  SEQUENCE_BITS = 10
+
+  MIN_LOGICAL_SHARD_ID = 0
+  MAX_LOGICAL_SHARD_ID = ~(-1 << LOGICAL_SHARD_ID_BITS)
+  LOGICAL_SHARD_ID_ALLOWED_RANGE = (MIN_LOGICAL_SHARD_ID..MAX_LOGICAL_SHARD_ID).freeze
+
+  MIN_DATA_TYPE = 0
+  MAX_DATA_TYPE = ~(-1 << DATA_TYPE_BITS)
+  DATA_TYPE_ALLOWED_RANGE = (MIN_DATA_TYPE..MAX_DATA_TYPE).freeze
+
+  MAX_SEQUENCE = ~(-1 << SEQUENCE_BITS)
 
   SEQUENCE_SHIFT = 0
-  LOGICAL_SHARD_ID_SHIFT = SEQUENCE_BITS
-  TIMESTAMP_SHIFT = SEQUENCE_BITS + LOGICAL_SHARD_ID_BITS
+  DATA_TYPE_SHIFT = SEQUENCE_BITS
+  LOGICAL_SHARD_ID_SHIFT = SEQUENCE_BITS + DATA_TYPE_BITS
+  TIMESTAMP_SHIFT = SEQUENCE_BITS + DATA_TYPE_BITS + LOGICAL_SHARD_ID_BITS
+
+  LOGICAL_SHARD_ID_RANGE_KEY = 'dogtag-generator-logical-shard-id-range'.freeze
 
-  LOGICAL_SHARD_ID_KEY = 'dogtag-generator-logical-shard-id'.freeze
+  def self.logical_shard_id_range=(logical_shard_id_range)
+    unless logical_shard_id_range.is_a? Range
+      raise ArgumentError, 'logical_shard_id_range must be a range'
+    end
+
+    unless (Dogtag::LOGICAL_SHARD_ID_ALLOWED_RANGE.to_a & logical_shard_id_range.to_a) == logical_shard_id_range.to_a
+      raise ArgumentError, "logical_shard_id_range is outside the allowed range of #{Dogtag::LOGICAL_SHARD_ID_ALLOWED_RANGE}"
+    end
 
-  def self.logical_shard_id=(logical_shard_id)
-    redis.set LOGICAL_SHARD_ID_KEY, logical_shard_id
+    if redis.exists LOGICAL_SHARD_ID_RANGE_KEY
+      redis.lset LOGICAL_SHARD_ID_RANGE_KEY, 0, logical_shard_id_range.min
+      redis.lset LOGICAL_SHARD_ID_RANGE_KEY, 1, logical_shard_id_range.max
+    else
+      redis.rpush LOGICAL_SHARD_ID_RANGE_KEY, logical_shard_id_range.min
+      redis.rpush LOGICAL_SHARD_ID_RANGE_KEY, logical_shard_id_range.max
+    end
   end
 
-  def self.generate_id
-    Generator.new(1).ids.first
+  def self.generate_id(data_type)
+    Generator.new(data_type, 1).ids.first
   end
 
-  def self.generate_ids(count)
+  def self.generate_ids(data_type, count)
     ids = []
 
     # The Lua script can't always return as many IDs as you may want. So we loop
     # until we have the exact amount.
     while ids.length < count
       initial_id_count = ids.length
-      ids += Generator.new(count - ids.length).ids
+      ids += Generator.new(data_type, count - ids.length).ids
 
       # Ensure the ids array keeps growing as infinite loop insurance
       return ids unless ids.length > initial_id_count

data/lib/dogtag/generator.rb

@@ -1,6 +1,18 @@
 module Dogtag
   class Generator
-    def initialize(count = 1)
+    def initialize(data_type, count = 1)
+      unless data_type.is_a? Integer
+        raise ArgumentError, 'data_type must be an integer'
+      end
+
+      unless Dogtag::DATA_TYPE_ALLOWED_RANGE.include? data_type
+        raise ArgumentError, "data_type is outside the allowed range of #{Dogtag::DATA_TYPE_ALLOWED_RANGE}"
+      end
+
+      raise ArgumentError, 'count must be an integer' unless count.is_a? Integer
+      raise ArgumentError, 'count must be a positive number' if count < 1
+
+      @data_type = data_type
       @count = count
     end
 
@@ -9,6 +21,7 @@ module Dogtag
         (
           shifted_timestamp |
           shifted_logical_shard_id |
+          shifted_data_type |
           (sequence << Dogtag::SEQUENCE_SHIFT)
         )
       end
@@ -16,19 +29,23 @@ module Dogtag
 
     private
 
-    attr_reader :count
+    attr_reader :data_type, :count
 
     def shifted_timestamp
       timestamp = Timestamp.from_redis(response.seconds, response.microseconds_part)
       timestamp.with_epoch(Dogtag::CUSTOM_EPOCH).milliseconds << Dogtag::TIMESTAMP_SHIFT
     end
 
+    def shifted_data_type
+      data_type << Dogtag::DATA_TYPE_SHIFT
+    end
+
     def shifted_logical_shard_id
       response.logical_shard_id << Dogtag::LOGICAL_SHARD_ID_SHIFT
     end
 
     def response
-      @response ||= Request.new(count).response
+      @response ||= Request.new(data_type, count).response
     end
   end
 end

data/lib/dogtag/id.rb

@@ -1,6 +1,7 @@
 module Dogtag
   class Id
     SEQUENCE_MAP = ~(-1 << Dogtag::SEQUENCE_BITS) << Dogtag::SEQUENCE_SHIFT
+    DATA_TYPE_MAP = ~(-1 << Dogtag::DATA_TYPE_BITS) << Dogtag::DATA_TYPE_SHIFT
     LOGICAL_SHARD_ID_MAP = (~(-1 << Dogtag::LOGICAL_SHARD_ID_BITS)) << Dogtag::LOGICAL_SHARD_ID_SHIFT
     TIMESTAMP_MAP = ~(-1 << Dogtag::TIMESTAMP_BITS) << Dogtag::TIMESTAMP_SHIFT
 
@@ -22,6 +23,10 @@ module Dogtag
       (id & LOGICAL_SHARD_ID_MAP) >> Dogtag::LOGICAL_SHARD_ID_SHIFT
     end
 
+    def data_type
+      (id & DATA_TYPE_MAP) >> Dogtag::DATA_TYPE_SHIFT
+    end
+
     def sequence
       (id & SEQUENCE_MAP) >> Dogtag::SEQUENCE_SHIFT
     end

data/lib/dogtag/mixins/redis.rb

@@ -2,7 +2,7 @@ module Dogtag
   module Mixins
     module Redis
       def redis
-        # TODO: Redis config
+        # TODO: Redis config for multiple servers
         @redis ||= ::Redis.new
       end
     end

data/lib/dogtag/request.rb

@@ -2,17 +2,19 @@ module Dogtag
   class Request
     include Dogtag::Mixins::Redis
 
-    MAX_SEQUENCE = ~(-1 << Dogtag::SEQUENCE_BITS)
-    MIN_LOGICAL_SHARD_ID = 1
-    MAX_LOGICAL_SHARD_ID = ~(-1 << Dogtag::LOGICAL_SHARD_ID_BITS)
     LUA_SCRIPT_PATH = 'lua/id-generation.lua'.freeze
     MAX_TRIES = 5
 
-    def initialize(count = 1)
+    def initialize(data_type, count = 1)
+      raise ArgumentError, 'data_type must be a number' unless data_type.is_a? Numeric
+      unless Dogtag::DATA_TYPE_ALLOWED_RANGE.include? data_type
+        raise ArgumentError, "data_type is outside the allowed range of #{Dogtag::DATA_TYPE_ALLOWED_RANGE}"
+      end
       raise ArgumentError, 'count must be a number' unless count.is_a? Numeric
       raise ArgumentError, 'count must be greater than zero' unless count > 0
 
       @tries = 0
+      @data_type = data_type
       @count = count
     end
 
@@ -22,7 +24,7 @@ module Dogtag
 
     private
 
-    attr_reader :count
+    attr_reader :data_type, :count
 
     def lua_script
       File.read(
@@ -31,12 +33,7 @@ module Dogtag
     end
 
     def lua_args
-      lua_args = [
-        MAX_SEQUENCE,
-        MIN_LOGICAL_SHARD_ID,
-        MAX_LOGICAL_SHARD_ID,
-        count
-      ]
+      lua_args = [ MAX_SEQUENCE, data_type, count ]
     end
 
     # NOTE: If too many requests come in inside of a millisecond the Lua script

data/lib/dogtag/version.rb

@@ -1,3 +1,3 @@
 module Dogtag
-  VERSION = '0.1.0'.freeze
+  VERSION = '1.0.0'.freeze
 end

data/lua/id-generation.lua

@@ -1,23 +1,51 @@
-local lock_key = 'dogtag-generator-lock'
-local sequence_key = 'dogtag-generator-sequence'
-local logical_shard_id_key = 'dogtag-generator-logical-shard-id'
+local logical_shard_id_range_key = 'dogtag-generator-logical-shard-id-range'
+local last_logical_shard_id_key = 'dogtag-generator-last-logical-shard-id'
 
 local max_sequence = tonumber(KEYS[1])
-local min_logical_shard_id = tonumber(KEYS[2])
-local max_logical_shard_id = tonumber(KEYS[3])
-local num_ids = tonumber(KEYS[4])
+local data_type = tonumber(KEYS[2])
+local num_ids = tonumber(KEYS[3])
+
+-- Allow one server to acts as multiple shards
+local logical_shard_id_range = redis.call('SORT', logical_shard_id_range_key)
+if next(logical_shard_id_range) == nil then
+  redis.log(redis.LOG_WARNING, 'Dogtag: ' .. logical_shard_id_range_key .. ' has not been set.')
+  return redis.error_reply('Dogtag: ' .. logical_shard_id_range_key .. ' has not been set.')
+end
+local logical_shard_id_min = tonumber(logical_shard_id_range[1])
+local logical_shard_id_max = tonumber(logical_shard_id_range[2])
+
+local logical_shard_id = nil
+if redis.call('EXISTS', last_logical_shard_id_key) == 0 then
+  logical_shard_id = logical_shard_id_min
+else
+  local last_shard_id = tonumber(redis.call('GET', last_logical_shard_id_key))
+
+  if last_shard_id >= logical_shard_id_max or last_shard_id < logical_shard_id_min then
+    logical_shard_id = logical_shard_id_min
+  else
+    logical_shard_id = last_shard_id + 1
+  end
+end
+
+redis.call('SET', last_logical_shard_id_key, logical_shard_id)
+
+--[[
+Scope lock and sequence keys to the specific data_type being requested.
+Ideally, we'd also use the logical_shard_id in the keys so that any per-millisecond limitations would only be per-shard,
+but unfortunately the whole "pure function" limitation keeps us from using a random shard_id here. The best solution may
+be to round robin the shard ID by incrementing a Redis key on each call.
+]]--
+local lock_key = 'dogtag-generator-lock-' .. logical_shard_id .. '-' .. data_type
+local sequence_key = 'dogtag-generator-sequence-' .. logical_shard_id .. '-' .. data_type
 
 if redis.call('EXISTS', lock_key) == 1 then
   redis.log(redis.LOG_NOTICE, 'Dogtag: Cannot generate ID, waiting for lock to expire.')
   return redis.error_reply('Dogtag: Cannot generate ID, waiting for lock to expire.')
 end
 
---[[
-Increment by a set number, this can
---]]
+-- Increment by a set number
 local end_sequence = redis.call('INCRBY', sequence_key, num_ids)
 local start_sequence = end_sequence - num_ids + 1
-local logical_shard_id = tonumber(redis.call('GET', logical_shard_id_key)) or -1
 
 if end_sequence >= max_sequence then
   --[[

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dogtag
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 1.0.0
 platform: ruby
 authors:
 - Adam Crownoble
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-05-09 00:00:00.000000000 Z
+date: 2017-05-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis