tamper 0.2.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 52b5b9d6757695b8df127daf7020a10feef37c27
+  data.tar.gz: 84a666eb3e032eed862a6b2deb4ff3baa315387e
+SHA512:
+  metadata.gz: 5022114100f3fa105def51905c8964979b79d7d78557312da37dd34f7744fd306c74dba3c608fa694227ef6b037bdd89cf5c1f7a046275e4fc98b5f1c6d1e164
+  data.tar.gz: 258e49d4efb9473f7e3b5d5abb064e6f3ca15cb90d75e400b56e570ce0151ff6ba05cffe0c038dee0584fd1602db313f6d4e36b1061d8e410f47c8893b747736

data/Gemfile

@@ -0,0 +1,9 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in tamper.gemspec
+gemspec
+
+gem 'rspec'
+gem 'byebug'
+gem 'json'
+gem 'rake'

data/README.md

@@ -0,0 +1,92 @@
+# Tamper Ruby encoder
+
+To encode data from ruby:
+
+  * Create a new `Tamper::PackSet`.
+  * Define attributes that should be packed with `PackSet#add_attribute`
+  * Dump the pack as JSON with `PackSet#to_json`
+
+### Init
+
+```ruby
+@pack_set = Tamper::PackSet.new
+```
+
+### Setting metadata on the pack
+
+This metadata is only necessary if you're buffing attributes,
+or your application requires extended info about the pack.
+
+```ruby
+@pack_set.meta = {
+  buffer_url: "http://",
+  buffer_callback: 'getDetails'
+}
+```
+
+### Adding a packed attribute
+
+These are the required attributes, you can set additional
+keys if required by your application:
+
+```ruby
+@pack_set.add_attribute(
+  name: :bucket,
+  possibilities: @project.buckets.map(&:downcase),
+  max_choices: 1
+)
+```
+
+### Adding a buffered attribute
+
+```ruby
+@pack_set.add_buffered_attribute(
+  name: 'name'
+)
+```
+
+### Encoding data
+
+** Important Note: ** the `id` attribute must be numeric.  Tamper cannot pack string IDs, since ExistencePack compression relies on contiguous ranges.  If the `id` is a string, it will be coerced to an int.
+
+#### If you can provide an array of hashes upfront
+
+
+```ruby
+# Keys can be either symbols or strings 
+data = [{ 'id' : '1', 'zip_code' : '06475' }, { 'id' : '1', 'zip_code' : '91647' }]
+
+@pack_set.pack!(data)
+```
+
+#### If you need to iterate (for example when doing a find_in_batches)
+
+```ruby
+@pack_set.build_pack(max_guid: max_guid) do |pack|
+  photo.each do |p|
+    pack << {
+      id: p.serial,
+      bucket: p.submission.bucket,
+      zip_code: p.zip_code,
+      state: p.submitter.state
+    }
+  end
+end
+```
+
+### Outputting pack
+
+```ruby
+puts @pack_set.to_json
+```
+
+### For more info
+
+See [protocol specs](https://github.com/newsdev/tamper/wiki/Packs) on the Tamper wiki.
+
+### Running tests
+
+ruby-tamper ships with:
+
+  * rspecs, which can be run with `bundle exec rspec`
+  * functional tests to generate output based on reference datasets.  Results are compared to the canonical-output in the root of the project. Run these with `bundle exec rake functional:test`

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+load "./functional_test/test.rake"

data/functional_test/config.json

@@ -0,0 +1,28 @@
+{
+  "attrs": [
+    {"attr_name": "gender",
+     "display_name": "Integer Pack",
+     "max_choices": 1,
+     "possibilities": ["male","female","cis"],
+     "filter_type": "category", 
+     "display_type": "string"},
+    {"attr_name": "bmp",
+     "display_name": "Bitmap Attr",
+     "max_choices": 8,
+     "possibilities": ["a","b","c","d","e","f","g","h"],
+     "filter_type": "category", 
+     "display_type": "string"},
+    {"attr_name": "unbmp",
+     "display_name": "Unaligned Bitmap Attr",
+     "max_choices": 2,
+     "possibilities": ["left","right","up","down"],
+     "filter_type": "category", 
+     "display_type": "string"},
+    {"attr_name": "num",
+     "display_name": "Numeric Attr",
+     "max_choices": 2,
+     "possibilities": ["1","2","3","4","5","6","7","8","9","10"],
+     "filter_type": "category", 
+     "display_type": "string"}
+  ]
+}

data/functional_test/test.rake

@@ -0,0 +1,95 @@
+require 'json'
+require 'tamper'
+
+# The test/datasets folder at the root of the project
+# contains a variety of reference datasets.
+#
+# `rake functional:generate` will write generate tamper output for each
+# test file and write the result to output/ruby-tamper.
+#
+# Use `rake functional:compare` to compare the outputs to the canonical version.
+namespace :functional do
+
+  desc "Generate functional outputs, then compare them"
+  task :test do
+    Rake::Task["functional:generate"].invoke
+    Rake::Task["functional:compare"].invoke
+  end
+
+  desc "Generate functional outputs for this gem using the test inputs."
+  task :generate do
+    base_path  = File.dirname(__FILE__)
+    config     = JSON.parse(File.read(File.join(base_path, 'config.json')), symbolize_names: true)
+    inputs     = Dir.glob(File.join(base_path, '..', '..', '..', 'test', 'datasets', '*.json'))
+    output_dir = File.join(base_path, 'output')
+
+    puts "Sweeping output dir..."
+    FileUtils.rm_f(File.join(output_dir, '*.json'))
+    FileUtils.mkdir_p(output_dir)
+
+    puts "Generating test output for ruby-tamper."
+
+    inputs.each do |input_file|
+      print "Packing #{File.basename(input_file)}... "
+      @pack_set = Tamper::PackSet.new
+      config[:attrs].each { |attr| @pack_set.add_attribute(attr) }
+
+      begin
+        data = JSON.parse(File.read(input_file))
+        @pack_set.pack!(data['items'], guid_attr: 'guid')
+        puts "Success!"
+      rescue Exception => e
+        puts; puts "ERROR: #{e.class} #{e.message}"
+        puts e.backtrace.select { |l| l.match('lib/tamper') }.join("\n")
+        exit(1)
+      end
+
+      File.open(File.join(output_dir, File.basename(input_file)), 'w') { |f| f.write(@pack_set.to_json) }
+    end
+  end
+
+  desc "Run this to diff output vs. canonical outputs."
+  task :compare do
+    base_path     = File.dirname(__FILE__)
+    reference_dir = File.join(base_path, '..', '..', '..', 'test', 'canonical-output')
+    output_files  = Dir.glob(File.join(base_path, 'output', '*.json'))
+    diffs = []
+    
+    output_files.each do |test_output|
+      test_file = File.basename(test_output)
+
+      puts "\nResults for: #{test_file}"
+      reference_data = JSON.parse(File.read(File.join(reference_dir, test_file)))
+
+      ruby_data = JSON.parse(File.read(test_output))
+
+      diffs << diff('existence', reference_data['existence']['pack'], ruby_data['existence']['pack'])
+
+      reference_data['attributes'].each do |reference_attr|
+        ruby_attr = ruby_data['attributes'].detect { |attr| attr['attr_name'] == reference_attr['attr_name'] }
+        ruby_attr.delete('max_guid')
+        reference_attr.delete('max_guid')
+        diffs << diff(reference_attr['attr_name'], reference_attr, ruby_attr)
+      end
+    end
+
+    puts
+    if diffs.any? { |d| d == false }
+      puts "**** There were errors in this test run."
+    else
+      puts "Functional tests OK!"
+    end
+  end
+
+  # Returns true if values were the same, false otherwise.
+  def diff(attr_name, var1, var2)
+    if var1 == var2
+      puts "    #{attr_name} is the same."
+      return true
+    else
+      puts "    ERROR on #{attr_name}!\n    reference was #{var1}\n    but ruby was #{var2}"
+      return false
+    end
+  end
+
+end

data/lib/tamper.rb

@@ -0,0 +1,15 @@
+require 'bitset'
+require 'base64'
+require 'json'
+require 'oj'
+
+require 'tamper/version'
+require 'tamper/pack_set'
+require 'tamper/pack'
+require 'tamper/existence_pack'
+require 'tamper/integer_pack'
+require 'tamper/bitmap_pack'
+
+module Tamper
+  # Your code goes here...
+end

data/lib/tamper/bitmap_pack.rb

@@ -0,0 +1,26 @@
+module Tamper
+  class BitmapPack < Pack
+
+    def encoding
+      :bitmap
+    end
+
+    def encode(idx, data)
+      choice_data = data[attr_name.to_sym] || data[attr_name.to_s]
+      choice_data = [choice_data] unless choice_data.is_a?(Array)
+
+      item_offset = idx * item_window_width
+      choice_data.each do |choice|
+        choice_offset = possibilities.index(choice.to_s)
+        @bitset[item_offset + choice_offset] = true if choice_offset
+      end
+    end
+
+    def initialize_pack!(max_guid, num_items)
+      @bit_window_width = 1
+      @item_window_width = possibilities.length
+      @bitset = Bitset.new(item_window_width * num_items)
+    end
+
+  end
+end

data/lib/tamper/existence_pack.rb

@@ -0,0 +1,101 @@
+module Tamper
+  class ExistencePack < Pack
+
+    def initialize
+      @output = ''
+      @current_chunk = ''
+      @last_guid   = 0
+      @run_counter = 0
+    end
+
+    def initialize_pack!(max_guid, num_items)
+    end
+
+    def encoding
+      :existence
+    end
+
+    def encode(guid)
+      guid_diff  = guid.to_i - @last_guid
+      guid_diff += 1 if @current_chunk.empty? && @output.empty? && guid.to_i > 0
+
+      if guid_diff == 1 || guid.to_i == 0  # guid is 1 step forward
+        @current_chunk << '1'
+        @run_counter += 1
+
+      elsif guid_diff <= 0  # somehow we went backwards or didn't change guid on iteration
+        raise ArgumentError, "Error: data was not sorted by GUID (got #{@last_guid}, then #{guid})!"
+    
+      elsif guid_diff > 40  # big gap, encode with skip control char
+        dump_keep(@current_chunk, @run_counter)
+
+        @output += control_code(:skip, guid_diff - 1)
+        @current_chunk = '1'
+        @run_counter   = 1
+
+      else # skips < 40 should just be encoded as '0'
+        if @run_counter > 40    # first check if a run came before this 0; if so dump it
+          dump_keep(@current_chunk, @run_counter)
+          @current_chunk = ''
+          @run_counter = 0
+        end
+
+        @current_chunk += ('0' * (guid_diff - 1))
+        @current_chunk << '1'
+        @run_counter = 1
+      end
+
+      @last_guid = guid.to_i
+    end
+
+    def finalize_pack!
+      dump_keep(@current_chunk, @run_counter)
+      raise "Encoding error, #{@output.length} is not an even number of bytes!" if @output.length % 8 > 0
+      @bitset = Bitset.from_s(@output)
+    end
+
+    def to_h
+      { encoding: encoding,
+        pack: encoded_bitset }
+    end
+
+
+    private
+    def dump_keep(chunk, run_len)
+      if run_len >= 40
+        dump_keep(chunk[0, chunk.length - run_len], 0)
+        @output += control_code(:run, run_len)
+      elsif !(chunk.nil? || chunk.empty?)
+        @output += control_code(:keep, chunk.length)
+        @output += chunk
+
+        # If the keep is not an even number of bytes, pad with 0 until it can be evenly packed
+        if (@output.length % 8) > 0
+          @output += '0' * (8 - (@output.length % 8))
+        end
+      end
+    end
+
+    def control_code(cmd, offset=0)
+      case cmd
+      when :keep
+        control_seq = '00000000'
+        bytes_to_keep = offset / 8
+        control_seq += bytes_to_keep.to_s(2).rjust(32)
+
+        remaining_bits = offset % 8
+        control_seq += remaining_bits.to_s(2).rjust(8)
+      when :skip
+        control_seq = '00000001'
+        control_seq += offset.to_s(2).rjust(32)
+      when :run
+        control_seq = '00000010'
+        control_seq += offset.to_s(2).rjust(32)
+      else
+        raise "Unknown control cmd '#{cmd}'!"
+      end
+      control_seq
+    end
+
+  end
+end

data/lib/tamper/integer_pack.rb

@@ -0,0 +1,43 @@
+module Tamper
+  class IntegerPack < Pack
+
+    def encoding
+      :integer
+    end
+
+    def encode(idx, data)
+      choice_data = data[attr_name.to_sym] || data[attr_name.to_s]
+      choice_data = [choice_data] unless choice_data.is_a?(Array)
+
+      (0...max_choices).each do |choice_idx|
+        choice_offset = (item_window_width * idx) + (bit_window_width * choice_idx)
+
+        value = choice_data[choice_idx]
+
+        # TODO: test and handle nil case
+        if possibility_idx = possibilities.index(value.to_s)
+          possibility_id =  possibility_idx + 1
+        else
+          possibility_id = 0
+        end
+
+        bit_code = possibility_id.to_i.to_s(2).split('') # converts to str binary representation
+        bit_code_length_pad = bit_window_width - bit_code.length
+        bit_code.each_with_index do |bit, bit_idx|
+          byebug if (choice_offset + bit_code_length_pad + bit_idx) == -1
+          @bitset[(choice_offset + bit_code_length_pad + bit_idx)] = bit == "1"
+        end
+      end
+
+    end
+
+    def initialize_pack!(max_guid, num_items)
+      @bit_window_width = Math.log2(possibilities.length + 1).ceil  # add 1 to possiblities.length for the implicit nil possiblity.
+      @bit_window_width = 1        if @bit_window_width == 0  # edge case: 1 possibility
+
+      @item_window_width = bit_window_width * max_choices
+      @bitset = Bitset.new(item_window_width * num_items)
+    end
+
+  end
+end

data/lib/tamper/pack.rb

@@ -0,0 +1,58 @@
+module Tamper
+  class Pack
+    attr_reader :attr_name, :possibilities, :max_choices, :encoding, :bitset
+
+    attr_reader :bit_window_width, :item_window_width, :bitset
+
+    attr_reader :max_guid
+
+    attr_accessor :meta
+
+    def initialize(attr_name, possibilities, max_choices)
+      @attr_name, @possibilities, @max_choices = attr_name, possibilities, max_choices
+      @meta = {}
+
+      raise ArgumentError, "Possibilities are empty for #{attr_name}!" if possibilities.nil? || possibilities.empty?
+      @possibilities.map!(&:to_s) # tamper values/possibilities should always be strings.
+    end
+
+    def self.build(attr_name, possibilities, max_choices)
+      if (max_choices * Math.log2(possibilities.length)) < possibilities.length
+        pack = IntegerPack
+      else
+        pack = BitmapPack
+      end
+
+      pack.new(attr_name, possibilities, max_choices)
+    end
+
+    def to_h
+      output = { encoding: encoding,
+                attr_name: attr_name,
+                possibilities: possibilities,
+                pack: encoded_bitset,
+                item_window_width: item_window_width,
+                bit_window_width: bit_window_width,
+                max_choices: max_choices }
+      output.merge(meta)
+    end
+
+    # Most packs do not implement this.
+    def finalize_pack!
+      data = @bitset.to_s
+      byte_length = data.length / 8
+      remaining_bits = data.length % 8
+
+      output  = byte_length.to_s(2).rjust(32)
+      output += remaining_bits.to_s(2).rjust(8)
+      output += data
+
+      @bitset = Bitset.from_s(output)
+    end
+
+    private
+    def encoded_bitset
+      Base64.strict_encode64(@bitset.marshal_dump[:data].unpack('b*').pack('B*')) if @bitset
+    end
+  end
+end

data/lib/tamper/pack_set.rb

@@ -0,0 +1,120 @@
+module Tamper
+  class PackSet
+
+    attr_accessor :meta, :existence_pack
+
+    def initialize(opts={})
+      @existence_pack = ExistencePack.new 
+      @attr_packs = {}
+      @buffered_attrs = {}
+      @meta = opts
+    end
+
+    def add_attribute(opts)
+      opts = opts.dup
+      [:attr_name, :possibilities, :max_choices].each do |required_opt|
+        raise ArgumentError, ":#{required_opt} is required when adding an attribute!" if !opts.key?(required_opt)
+      end
+
+      name          = opts.delete(:attr_name)
+      possibilities = opts.delete(:possibilities).compact
+      max_choices   = opts.delete(:max_choices)
+
+      pack      = Pack.build(name, possibilities, max_choices)
+      pack.meta = opts
+      @attr_packs[name.to_sym] = pack
+      pack
+    end
+
+
+    # Buffered attributes will not be packed, but their metadata will be included in the PackSet's JSON
+    # representation.  Clients will expect these attrs to be available via the <tt>buffer_url</tt>.
+    def add_buffered_attribute(opts)
+      opts = opts.dup
+      raise ArgumentError, ":attr_name is required when adding a buffered attribute!" if !opts.key?(:attr_name)
+
+      attr_name   = opts.delete(:attr_name)
+      @buffered_attrs[attr_name.to_sym] = { attr_name: attr_name }.merge(opts)
+    end
+
+    def attributes
+      @attr_packs.keys
+    end
+
+    def pack_for(attr)
+      @attr_packs[attr]
+    end
+
+    def pack!(data, opts={})
+      opts = opts.dup
+      opts[:guid_attr] ||= 'id'
+      opts[:max_guid]  ||= (data.last[opts[:guid_attr].to_sym] || data.last[opts[:guid_attr].to_s])
+      opts[:num_items] ||= data.length
+
+      build_pack(opts) do |p|
+        data.each { |d| p << d }
+      end
+    end
+
+    def build_pack(opts={}, &block)
+      guid_attr = opts[:guid_attr] || 'id'
+      packs = @attr_packs.values
+      max_guid  = opts[:max_guid]
+      num_items = opts[:num_items]
+
+      [:num_items, :max_guid].each do |required_opt|
+        raise ArgumentError, "You must specify :#{required_opt} to start building a pack!" if !opts.key?(required_opt)
+      end
+
+      existence_pack.initialize_pack!(max_guid, num_items)
+      packs.each { |p| p.initialize_pack!(max_guid, num_items) }
+
+      idx = 0
+      packer = ->(d) {
+        guid = d[guid_attr.to_sym] || d[guid_attr.to_s]
+        existence_pack.encode(guid)
+        packs.each { |p| p.encode(idx, d) }
+        idx += 1
+      }
+      packer.instance_eval { alias :<< :call; alias :add :call }
+
+      yield(packer)
+
+      existence_pack.finalize_pack!
+      packs.each { |p| p.finalize_pack! }
+    end
+
+    def build_unordered_pack(opts={}, &block)
+      guid_attr = opts[:guid_attr] || 'id'
+      data = {}
+
+      extractor = ->(d) {
+        guid = d[guid_attr.to_sym] || d[guid_attr.to_s]
+        data[guid] = d
+      }
+      extractor.instance_eval { alias :<< :call; alias :add :call }
+
+      yield(extractor)
+
+      sorted_guids = data.keys.sort
+      sorted_data = sorted_guids.map { |guid| data[guid] }
+      pack!(sorted_data, opts)
+    end
+
+    def to_hash(opts={})
+      output = {
+        version: '2.1',
+        existence: @existence_pack.to_h,
+        attributes: @attr_packs.values.map { |p| p.to_h }
+      }
+
+      output[:attributes] += @buffered_attrs.values
+      output.merge!(meta)
+      output
+    end
+
+    def to_json(opts={})
+      Oj.dump self, { mode: :compat }
+    end
+  end
+end

data/lib/tamper/version.rb

@@ -0,0 +1,3 @@
+module Tamper
+  VERSION = "0.2.2"
+end

data/spec/bitmap_pack_spec.rb

@@ -0,0 +1,48 @@
+require 'spec_helper'
+
+describe Tamper::BitmapPack do
+
+  before do
+    @data = [
+      { 'id' => 0, 'color' => ['yellow','blue'] },
+      { 'id' => 1, 'color' => ['blue', 'yellow'] },
+      { 'id' => 2, 'color' => ['red'] },
+    ]
+
+    @pack_set = Tamper::PackSet.new
+    @pack_set.add_attribute(attr_name: :color,
+                            possibilities: ['yellow', 'red', 'blue', 'purple'],
+                            max_choices: 2)
+    @pack_set.pack!(@data)
+
+    @color_pack = @pack_set.pack_for(:color)
+    @color_pack.should be_a(Tamper::BitmapPack)
+  end
+
+  its "item_window_width is equal to the number of possibilities, since multiple bits can be flipped for each item" do
+    @color_pack.item_window_width.should == ['yellow','red','blue','purple'].length
+  end
+
+  its "length is equal to the number of choices * the number of items" do
+    @color_pack.bitset.size.should == (4 * 3) + 32 + 8
+  end
+
+  it "encodes the length at the front of the pack" do
+    @color_pack.bitset.to_s[0,32].to_i(2).should == 1
+    @color_pack.bitset.to_s[32,8].to_i(2).should == 4
+  end
+
+  it "flips a bit true for each selected choice" do
+    str_bits = @color_pack.bitset.to_s
+
+    # first item should have idx 0 set to true since it's yellow, idx 2 since it's blue
+    str_bits[40,4].should == '1010'
+
+    # second item should have the same, order that possibilities are in does not matter
+    str_bits[44,4].should == '1010'
+
+    # last item should have pos 2 set to true since it's red
+    str_bits[48,4].should == '0100'
+  end
+
+end

data/spec/existence_pack_spec.rb

@@ -0,0 +1,157 @@
+require 'spec_helper'
+
+describe Tamper::ExistencePack do
+
+  describe "with continguous ids" do
+    before do
+      @data = [
+        { 'id' => 0 },
+        { 'id' => 1 },
+        { 'id' => 2 }
+      ]
+
+      @pack_set = Tamper::PackSet.new
+      @pack_set.pack!(@data)
+
+      @existence_pack = @pack_set.existence_pack
+      @existence_pack.should be_a(Tamper::ExistencePack)
+      @bits = @existence_pack.bitset.to_s
+    end
+
+    it "sets a control code at the start to keep all ids" do
+      @bits[0,8].should == '00000000'
+    end
+
+    it "includes 0 bytes in existence bitmap" do
+      @bits[8,32].to_i(2).should == 0
+    end
+
+    it "includes 3 remainder bits in existence bitmap" do
+      @bits[40,8].to_i(2).should == 3
+    end
+
+    it "sets all ids as existing, then pads with zeroes" do
+      @bits[48,8].should == '11100000'
+    end
+  end
+
+  describe "with gaps in ids" do
+    before do
+      @data = [
+        { 'id' => 0 },
+        { 'id' => 1 },
+        { 'id' => 2 },
+        { 'id' => 6 },
+        { 'id' => 60 },
+        { 'id' => 61 },
+        { 'id' => 62 },
+      ]
+
+      @pack_set = Tamper::PackSet.new
+      @pack_set.pack!(@data)
+
+      @existence_pack = @pack_set.existence_pack
+      @existence_pack.should be_a(Tamper::ExistencePack)
+      @bits = @existence_pack.bitset.to_s
+    end
+
+    it "keeps all ids with gaps < 20 in a contiguous set" do
+      @bits[0,8].should == '00000000' # keep code
+      @bits[8,32].to_i(2).should == 0 # keep 0 bytes of data
+      @bits[40,8].to_i(2).should == 7 # keep 7 remainder bits
+      @bits[48,8].to_s.should == '11100010' # encode 1-6
+    end
+
+    it "sets a skip control char if the gap is > 40" do
+      @bits[56,8].should == '00000001'
+      @bits[64,32].to_i(2).should == 53
+    end
+
+    it "sets a keep control char after the gap" do
+      @bits[96,8].should == '00000000'  # keep
+      @bits[104,32].to_i(2).should == 0  # keep 0 bytes
+      @bits[136,8].to_i(2).should == 3   # keep 3 remainder bits
+    end
+
+    it "sets bits correctly after a skipped gap" do
+      @bits[144,8].to_s.should == '11100000'
+    end
+  end
+
+  describe "with a run of ids" do
+    before do
+      guids = [(0..59).to_a, 67,68].flatten
+      @data = guids.map { |g| { 'id' => g } }
+      @pack_set = Tamper::PackSet.new
+      @pack_set.pack!(@data)
+
+      @existence_pack = @pack_set.existence_pack
+      @existence_pack.should be_a(Tamper::ExistencePack)
+      @bits = @existence_pack.bitset.to_s
+    end
+
+    it "sets a run for the continguous ids" do
+      @bits[0,8].should == '00000010' # run code
+      @bits[8,32].to_i(2).should == 60 # run of 60 guids
+    end
+
+    it "sets a keep control char after the run" do
+      @bits[40,8].should == '00000000'  # keep
+      @bits[48,32].to_i(2).should == 1  # keep 0 bytes
+      @bits[80,8].to_i(2).should  == 1  # keep 1 remaining bit
+    end
+
+    it "sets bits correctly after a skipped gap" do
+      @bits[88,16].to_s.should == '00000001' + '10000000'
+    end
+  end
+
+  describe "with a guid sequence that starts with a skip" do
+    before do
+      @data = [
+        { 'id' => 100 },
+        { 'id' => 101 },
+        { 'id' => 102 }
+      ]
+
+      @pack_set = Tamper::PackSet.new
+      @pack_set.pack!(@data)
+
+      @existence_pack = @pack_set.existence_pack
+      @existence_pack.should be_a(Tamper::ExistencePack)
+      @bits = @existence_pack.bitset.to_s
+    end
+
+    it "sets a skip control code at the start of the set" do
+      @bits[0,8].should == '00000001'  # skip
+      @bits[8,32].to_i(2).should == 100 # skip 99 guids
+    end
+  end
+
+  describe "the example from the README" do
+    before do
+
+      test_guids = [0,1,2,3,6,10,12,14,16,17,18,19,20,21,22,23,25,31,32,97]
+      @data = test_guids.map { |guid| { 'id' => guid }}
+
+      @pack_set = Tamper::PackSet.new
+      @pack_set.pack!(@data)
+
+      @existence_pack = @pack_set.existence_pack
+      @existence_pack.should be_a(Tamper::ExistencePack)
+      @bits = @existence_pack.bitset.to_s
+    end
+
+    it "works" do
+      example_data = <<EOS
+    00000000 00000000 00000000 00000000 00000100 00000001
+    11110010 00101010 11111111 01000001 10000000 00000001
+    00000000 00000000 00000000 01000000 00000000 00000000 
+    00000000 00000000 00000000 00000001 10000000 
+EOS
+      example_data.gsub!(/[ \n]/, '')
+      @bits.should == example_data
+    end
+  end
+
+end

data/spec/integer_pack_spec.rb

@@ -0,0 +1,85 @@
+require 'spec_helper'
+
+describe Tamper::IntegerPack do
+
+  describe "with a large number of possibilities" do
+    before do
+      @data = [
+        { 'id' => 0, 'category_id' => '14' },
+        { 'id' => 1, 'category_id' => '21' },
+        { 'id' => 2, 'category_id' => '46' },
+      ]
+
+      @possibilities = (0...50).to_a.map(&:to_s)
+
+      @pack_set = Tamper::PackSet.new
+      @pack_set.add_attribute(attr_name: :category_id,
+                              possibilities: @possibilities,
+                              max_choices: 1)
+      @pack_set.pack!(@data)
+
+      @category_pack = @pack_set.pack_for(:category_id)
+      @category_pack.should be_a(Tamper::IntegerPack)
+    end
+
+    its "bit_window_width is equal to the number of bits required to represent the max possibility id as an int, + 1 for the implicit nil poss." do
+      @category_pack.bit_window_width.should == @possibilities.length.to_s(2).length
+    end
+
+    its "item_window_width is to the bit_window_width" do
+      @category_pack.item_window_width.should == @category_pack.bit_window_width
+    end
+
+    its "overall length is equal to the item_window_width * number of items" do
+      @category_pack.bitset.size.should == @category_pack.item_window_width * 3 + 32 + 8
+    end
+
+    it "encodes the length at the front of the pack" do
+      @category_pack.bitset.to_s[0,32].to_i(2).should == 2
+      @category_pack.bitset.to_s[32,8].to_i(2).should == 2
+    end
+
+
+    it "encodes the choice id in binary for each item" do
+      str_bits = @category_pack.bitset.to_s
+
+      # first item should be packed to represent 14
+      # note that this is +1 since possibility 0 represents 'no choice'
+      str_bits[40,6].should == 15.to_s(2).rjust(6, '0')
+
+      # next 21
+      str_bits[46,6].should == 22.to_s(2).rjust(6, '0')
+
+      # last item should be packed to represent 46
+      str_bits[52,6].should == 47.to_s(2).rjust(6, '0')
+    end
+
+  end
+
+  describe "with only two possibilities" do
+    before do
+      @data = [
+        { 'id' => 0, 'gender' => 'female' },
+        { 'id' => 1, 'gender' => 'male' }
+      ]
+
+      @possibilities = %W(female male)
+
+      @pack_set = Tamper::PackSet.new
+      @pack_set.add_attribute(attr_name: :gender,
+                              possibilities: @possibilities,
+                              max_choices: 1)
+      @pack_set.pack!(@data)
+
+      @category_pack = @pack_set.pack_for(:gender)
+      @category_pack.should be_a(Tamper::IntegerPack)
+    end
+
+    its "bit_window_width is equal to the number of bits required to represent the max possibility id, + 1 for the implicit nil poss." do
+      @category_pack.bit_window_width.should == 2
+    end
+  end
+
+
+
+end

data/spec/pack_set_spec.rb

@@ -0,0 +1,64 @@
+require 'spec_helper'
+
+describe Tamper::PackSet do
+  before do
+    @data = [
+      { 'id' => 0, 'category_id' => '14' },
+      { 'id' => 1, 'category_id' => '21' },
+      { 'id' => 2, 'category_id' => '46' },
+    ]
+
+    @possibilities = (0...50).to_a.map(&:to_s)
+
+    @pack_set = Tamper::PackSet.new(default_thumb: 'thumb_url')
+    @pack_set.add_attribute(attr_name: :category_id,
+                            possibilities: @possibilities,
+                            max_choices: 1,
+                            filter_type: 'category')
+  end
+
+  it "packs data using a block" do
+    @pack_set.build_pack(max_guid: 2, num_items: @data.length) do |p|
+      @data.each { |d| p << d }
+    end
+
+    bits = @pack_set.existence_pack.bitset.to_s
+    bits[48,8].to_s.should == '11100000' # encode 1-3
+  end
+
+  it "saves additional attributes as pack.meta" do
+    @pack_set.pack_for(:category_id).meta.should == { filter_type: 'category' }
+  end
+
+  it "includes metadata passed at init as a top-level attribute in JSON" do
+    @pack_set.to_hash[:default_thumb].should == 'thumb_url'
+  end
+
+  it "jsonifies using Oj" do
+    @pack_set.to_json.is_a?(String).should be_true
+  end
+
+  it "can pack unordered data using a block" do
+    @data.first['id'] = 4
+
+    @pack_set.build_unordered_pack(max_guid: 4) do |p|
+      @data.each { |d| p << d }
+    end
+
+    bits = @pack_set.existence_pack.bitset.to_s
+    bits[48,8].to_s.should == '01101000' # encode 1-3
+  end
+
+  it "includes buffered attrs as part of the JSON attributes array" do
+    @pack_set.add_buffered_attribute(
+      attr_name: 'caption',
+      display_name: 'Display Caption',
+      filter_type: 'none'
+    )
+
+    caption_attr = @pack_set.to_hash[:attributes].detect { |attr| attr[:attr_name] ==  'caption' }
+    caption_attr.should_not be_nil
+    caption_attr[:display_name].should == 'Display Caption'
+  end
+
+end

data/spec/pack_spec.rb

@@ -0,0 +1,18 @@
+require 'spec_helper'
+describe Tamper::Pack do
+
+  describe ".build" do
+
+    # Some sample situations...
+    describe "with a large number of possibilities" do
+      it "creates an IntegerPack when only a single choice is allowed" do
+        Tamper::Pack.build(:category_id, (0...50).to_a, 1).should be_a(Tamper::IntegerPack)
+      end
+
+      it "creates a BitmapPack when there are a large number of choices allowed" do
+        Tamper::Pack.build(:category_id, (0...50).to_a, 50).should be_a(Tamper::BitmapPack)
+      end
+    end
+  end
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,10 @@
+require 'rubygems'
+require 'bundler/setup'
+
+require 'bitset'
+require 'tamper'
+require 'byebug'
+
+RSpec.configure do |config|
+  # some (optional) config here
+end

data/tamper.gemspec

@@ -0,0 +1,26 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "tamper/version"
+
+Gem::Specification.new do |s|
+  s.name        = "tamper"
+  s.version     = Tamper::VERSION
+  s.authors     = ["Ben Koski"]
+  s.email       = ["bkoski@nytimes.com"]
+  s.homepage    = ""
+  s.summary     = %q{Serialize defined-option attrs into miminalist binary arrays}
+  s.description = %q{Serialize defined-option attrs into miminalist binary arrays}
+
+  s.rubyforge_project = "tamper"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_dependency 'bitset'
+  s.add_dependency 'oj'
+  # specify any dependencies here; for example:
+  # s.add_development_dependency "rspec"
+  # s.add_runtime_dependency "rest-client"
+end

metadata

@@ -0,0 +1,96 @@
+--- !ruby/object:Gem::Specification
+name: tamper
+version: !ruby/object:Gem::Version
+  version: 0.2.2
+platform: ruby
+authors:
+- Ben Koski
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-04-16 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bitset
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: oj
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Serialize defined-option attrs into miminalist binary arrays
+email:
+- bkoski@nytimes.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- README.md
+- Rakefile
+- functional_test/config.json
+- functional_test/test.rake
+- lib/tamper.rb
+- lib/tamper/bitmap_pack.rb
+- lib/tamper/existence_pack.rb
+- lib/tamper/integer_pack.rb
+- lib/tamper/pack.rb
+- lib/tamper/pack_set.rb
+- lib/tamper/version.rb
+- spec/bitmap_pack_spec.rb
+- spec/existence_pack_spec.rb
+- spec/integer_pack_spec.rb
+- spec/pack_set_spec.rb
+- spec/pack_spec.rb
+- spec/spec_helper.rb
+- tamper.gemspec
+homepage: ''
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: tamper
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: Serialize defined-option attrs into miminalist binary arrays
+test_files:
+- spec/bitmap_pack_spec.rb
+- spec/existence_pack_spec.rb
+- spec/integer_pack_spec.rb
+- spec/pack_set_spec.rb
+- spec/pack_spec.rb
+- spec/spec_helper.rb