unique_content_set 0.0.1

data/LICENSE

@@ -0,0 +1,22 @@
+The MIT License
+
+Copyright (c) Rick Olson
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+

data/README.md

@@ -0,0 +1,42 @@
+# Unique Content Set
+
+Check for uniquely created content in a Redis set.
+
+## INSTALL
+
+    gem install unique_content_set
+
+## USAGE
+
+    # uses a Redis Set named something like "unique:5:messages"
+    set = UniqueContentSet.new user.id, :messages
+
+    if set.add(@message.body, @message.created_at)
+      # this message body has been posted before, do something!
+    end
+
+    if set.exist?(@message.body)
+      # this message body has been posted before, do something!
+    end
+
+    # Purge old message content.
+    set.delete_before(1.month.ago)
+
+## Contribute
+
+If you'd like to hack on UniqueContentSet, start by forking the repo on GitHub:
+
+`https://github.com/technoweenie/unique_content_set`
+
+The best way to get your changes merged back into core is as follows:
+
+* Clone down your fork
+* Create a thoughtfully named topic branch to contain your change
+* Hack away
+* Add tests and make sure everything still passes by running rake
+* If you are adding new functionality, document it in the README
+* Do not change the version number, I will do that on my end
+* If necessary, rebase your commits into logical chunks, without errors
+* Push the branch up to GitHub
+* Send a pull request to the `technoweenie/unique_content_set` project.
+

data/Rakefile

@@ -0,0 +1,135 @@
+require 'rubygems'
+require 'rake'
+require 'date'
+
+#############################################################################
+#
+# Helper functions
+#
+#############################################################################
+
+def name
+  @name ||= Dir['*.gemspec'].first.split('.').first
+end
+
+def version
+  line = File.read("lib/#{name}.rb")[/^\s*VERSION\s*=\s*.*/]
+  line.match(/.*VERSION\s*=\s*['"](.*)['"]/)[1]
+end
+
+def date
+  Date.today.to_s
+end
+
+def rubyforge_project
+  name
+end
+
+def gemspec_file
+  "#{name}.gemspec"
+end
+
+def gem_file
+  "#{name}-#{version}.gem"
+end
+
+def replace_header(head, header_name)
+  head.sub!(/(\.#{header_name}\s*= ').*'/) { "#{$1}#{send(header_name)}'"}
+end
+
+#############################################################################
+#
+# Standard tasks
+#
+#############################################################################
+
+task :default => :test
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/*_test.rb'
+  test.verbose = true
+end
+
+desc "Open an irb session preloaded with this library"
+task :console do
+  sh "irb -rubygems -r ./lib/#{name}.rb"
+end
+
+#############################################################################
+#
+# Custom tasks (add your own tasks here)
+#
+#############################################################################
+
+
+
+#############################################################################
+#
+# Packaging tasks
+#
+#############################################################################
+
+desc "Create tag v#{version} and build and push #{gem_file} to Rubygems"
+task :release => :build do
+  unless `git branch` =~ /^\* master$/
+    puts "You must be on the master branch to release!"
+    exit!
+  end
+  sh "git commit --allow-empty -a -m 'Release #{version}'"
+  sh "git tag v#{version}"
+  sh "git push origin master"
+  sh "git push origin v#{version}"
+  sh "gem push pkg/#{name}-#{version}.gem"
+end
+
+desc "Build #{gem_file} into the pkg directory"
+task :build => :gemspec do
+  sh "mkdir -p pkg"
+  sh "gem build #{gemspec_file}"
+  sh "mv #{gem_file} pkg"
+end
+
+desc "Generate #{gemspec_file}"
+task :gemspec => :validate do
+  # read spec file and split out manifest section
+  spec = File.read(gemspec_file)
+  head, manifest, tail = spec.split("  # = MANIFEST =\n")
+
+  # replace name version and date
+  replace_header(head, :name)
+  replace_header(head, :version)
+  replace_header(head, :date)
+  #comment this out if your rubyforge_project has a different name
+  replace_header(head, :rubyforge_project)
+
+  # determine file list from git ls-files
+  files = `git ls-files`.
+    split("\n").
+    sort.
+    reject { |file| file =~ /^\./ }.
+    reject { |file| file =~ /^(rdoc|pkg)/ }.
+    map { |file| "    #{file}" }.
+    join("\n")
+
+  # piece file back together and write
+  manifest = "  s.files = %w[\n#{files}\n  ]\n"
+  spec = [head, manifest, tail].join("  # = MANIFEST =\n")
+  File.open(gemspec_file, 'w') { |io| io.write(spec) }
+  puts "Updated #{gemspec_file}"
+end
+
+desc "Validate #{gemspec_file}"
+task :validate do
+  libfiles = Dir['lib/*'] - ["lib/#{name}.rb", "lib/#{name}"]
+  unless libfiles.empty?
+    puts "Directory `lib` should only contain a `#{name}.rb` file and `#{name}` dir."
+    exit!
+  end
+  unless Dir['VERSION*'].empty?
+    puts "A `VERSION` file at root level violates Gem best practices."
+    exit!
+  end
+end
+

data/lib/unique_content_set.rb

@@ -0,0 +1,71 @@
+require 'digest/sha1'
+
+# Stores a set of unique content.  This is used to check if new content being
+# added has been seen already.
+class UniqueContentSet
+  VERSION = '0.0.1'
+
+  class << self
+    # Sets are stored in Redis.
+    attr_accessor :redis,
+
+    # Prefix all Redis keys with a certain value.
+      :redis_prefix
+  end
+
+  self.redis_prefix = "unique"
+
+  attr_reader :key
+
+  def initialize(*args)
+    @redis = self.class.redis
+
+    # Turn the given args into a redis key, with pieces separated by ':'.
+    args.unshift self.class.redis_prefix
+    args.compact!
+    args.map! { |a| a.to_s }
+    @key = args * ":"
+  end
+
+  # Public: Adds the given content to the current set, scored by the
+  # given time.
+  #
+  # content - String content to add to the set.
+  # time    - The current Time the content was created.
+  #
+  # Returns true if this is the first occurence of the content, or false.
+  def add(content, time = Time.now)
+    @redis.zadd(@key, time.to_i, member_from(content))
+  end
+
+  # Public: Looks for the given content in the current set.
+  #
+  # content - String content that is being checked.
+  #
+  # Returns true if the content is a member of the set, or false.
+  def exist?(content)
+    !!@redis.zscore(@key, member_from(content))
+  end
+
+  # Public: Removes content posted before the given time.
+  #
+  # time - The latest Time that should be purged from the set.
+  #
+  # Returns a Fixnum of the number of removed entries.
+  def delete_before(time)
+    @redis.zremrangebyscore(@key, 0, time.to_i+1)
+  end
+
+private
+  # Encodes the content into a value that can be used to quickly check
+  # uniqueness in the set.
+  #
+  # content - The String content.
+  #
+  # Returns a String of the the size of the content, plus a SHA of the content,
+  # separated by a colon.
+  def member_from(content)
+    content = content.to_s
+    '%d:%s' % [content.size, Digest::SHA1.hexdigest(content)]
+  end
+end

data/test/unique_content_set_test.rb

@@ -0,0 +1,42 @@
+require File.expand_path("../../lib/unique_content_set", __FILE__)
+require 'test/unit'
+require 'rubygems'
+require 'redis'
+
+UniqueContentSet.redis = Redis.new
+UniqueContentSet.redis.select ENV['REDIS_DB'] || 7
+
+class UniqueContentSetTest < Test::Unit::TestCase
+  def setup
+    UniqueContentSet.redis.flushdb
+    @set  = UniqueContentSet.new :abc
+    @set.add 'abc', 1
+  end
+
+  def test_adding_unique_content
+    assert_equal true, @set.add('def')
+  end
+
+  def test_adding_repeated_content
+    assert_equal false, @set.add('abc')
+  end
+
+  def test_checking_content_existence
+    assert  @set.exist?('abc')
+    assert !@set.exist?('def')
+  end
+
+  def test_removing_old_content
+    @set.add 'def', 2
+    @set.add 'ghi', 3
+    assert @set.exist?('abc')
+    assert @set.exist?('def')
+    assert @set.exist?('ghi')
+
+    @set.delete_before(1)
+
+    assert !@set.exist?('abc')
+    assert !@set.exist?('def')
+    assert  @set.exist?('ghi')
+  end
+end

data/unique_content_set.gemspec

@@ -0,0 +1,76 @@
+## This is the rakegem gemspec template. Make sure you read and understand
+## all of the comments. Some sections require modification, and others can
+## be deleted if you don't need them. Once you understand the contents of
+## this file, feel free to delete any comments that begin with two hash marks.
+## You can find comprehensive Gem::Specification documentation, at
+## http://docs.rubygems.org/read/chapter/20
+Gem::Specification.new do |s|
+  s.specification_version = 2 if s.respond_to? :specification_version=
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.rubygems_version = '1.3.5'
+
+  ## Leave these as is they will be modified for you by the rake gemspec task.
+  ## If your rubyforge_project name is different, then edit it and comment out
+  ## the sub! line in the Rakefile
+  s.name              = 'unique_content_set'
+  s.version           = '0.0.1'
+  s.date              = '2011-03-28'
+  s.rubyforge_project = 'unique_content_set'
+
+  ## Make sure your summary is short. The description may be as long
+  ## as you like.
+  s.summary     = "Short description used in Gem listings."
+  s.description = "Long description. Maybe copied from the README."
+
+  ## List the primary authors. If there are a bunch of authors, it's probably
+  ## better to set the email to an email list or something. If you don't have
+  ## a custom homepage, consider using your GitHub URL or the like.
+  s.authors  = ["Rick Olson"]
+  s.email    = 'technoweenie@gmail.com'
+  s.homepage = 'http://github.com/technoweenie/unique_content_set'
+
+  ## This gets added to the $LOAD_PATH so that 'lib/NAME.rb' can be required as
+  ## require 'NAME.rb' or'/lib/NAME/file.rb' can be as require 'NAME/file.rb'
+  s.require_paths = %w[lib]
+
+  ## This sections is only necessary if you have C extensions.
+  #s.require_paths << 'ext'
+  #s.extensions = %w[ext/extconf.rb]
+
+  ## If your gem includes any executables, list them here.
+  #s.executables = ["name"]
+  #s.default_executable = 'name'
+
+  ## Specify any RDoc options here. You'll want to add your README and
+  ## LICENSE files to the extra_rdoc_files list.
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.extra_rdoc_files = %w[README.md LICENSE]
+
+  ## List your runtime dependencies here. Runtime dependencies are those
+  ## that are needed for an end user to actually USE your code.
+  s.add_dependency('redis', ["~> 2.1.0", "~> 2.0.0"])
+
+  ## List your development dependencies here. Development dependencies are
+  ## those that are only needed during development
+  #s.add_development_dependency('DEVDEPNAME', [">= 1.1.0", "< 2.0.0"])
+
+  ## Leave this section as-is. It will be automatically generated from the
+  ## contents of your Git repository via the gemspec task. DO NOT REMOVE
+  ## THE MANIFEST COMMENTS, they are used as delimiters by the task.
+  # = MANIFEST =
+  s.files = %w[
+    LICENSE
+    README.md
+    Rakefile
+    lib/unique_content_set.rb
+    test/unique_content_set_test.rb
+    unique_content_set.gemspec
+  ]
+  # = MANIFEST =
+
+  ## Test files will be grabbed from the file list. Make sure the path glob
+  ## matches what you actually use.
+  s.test_files = s.files.select { |path| path =~ /^test\/.*_test\.rb/ }
+end
+
+

metadata

@@ -0,0 +1,96 @@
+--- !ruby/object:Gem::Specification 
+name: unique_content_set
+version: !ruby/object:Gem::Version 
+  hash: 29
+  prerelease: false
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- Rick Olson
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-03-28 00:00:00 -07:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: redis
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 11
+        segments: 
+        - 2
+        - 1
+        - 0
+        version: 2.1.0
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 15
+        segments: 
+        - 2
+        - 0
+        - 0
+        version: 2.0.0
+  type: :runtime
+  version_requirements: *id001
+description: Long description. Maybe copied from the README.
+email: technoweenie@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README.md
+- LICENSE
+files: 
+- LICENSE
+- README.md
+- Rakefile
+- lib/unique_content_set.rb
+- test/unique_content_set_test.rb
+- unique_content_set.gemspec
+has_rdoc: true
+homepage: http://github.com/technoweenie/unique_content_set
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: unique_content_set
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 2
+summary: Short description used in Gem listings.
+test_files: 
+- test/unique_content_set_test.rb