tweet_compressor 0.8.2

data/README.md

@@ -0,0 +1,125 @@
+# tweet\_compressor
+
+## Copyright and Licensing
+
+### Copyright Notice
+
+The copyright for the software, documentation, and associated files are
+held by the author.
+
+    Copyright 2013 Todd A. Jacobs
+    All rights reserved.
+
+The AUTHORS file is also included in the source tree.
+
+### Software License
+
+![GPLv3 Logo](http://www.gnu.org/graphics/gplv3-88x31.png)
+
+The software is licensed under the
+[GPLv3](http://www.gnu.org/copyleft/gpl.html). The LICENSE file is also
+included in the source tree.
+
+### README License
+
+![Creative Commons BY-NC-SA
+Logo](http://i.creativecommons.org/l/by-nc-sa/3.0/us/88x31.png)
+
+This README is licensed under the [Creative Commons
+Attribution-NonCommercial-ShareAlike 3.0 United States
+License](http://creativecommons.org/licenses/by-nc-sa/3.0/us/).
+
+## Purpose
+
+tweet\_compressor is Ruby gem that performs successive text
+transformations in order to shrink input text below Twitter's
+140-character limit while preserving the integrity of hashtags and
+links.
+
+## Features
+
+- Treats hashtags as sacrosanct.
+- Relies on Twitter to shorten URLs for you, counting URLs as 20
+  characters.
+- Skips shortening stages whenever the character length drops below 140.
+- Remains vaguely intelligible even under heavy compression.
+
+## Caveats and Limitations
+
+1. The gem performs text transformations; it's not a full parser.
+2. Some of the transformations may be naive or rely on brute force to
+   get the job done. YMMV.
+3. No sanity checking is performed on the semantics of the output text.
+   It Works for Me™, but it's not a substitute for applying common
+   sense and a keen eye to your tweets before posting on Twitter.
+4. Works best when you only need to trim a handful of characters. If
+   you're vastly over the limit, readability suffers as compression gets
+   tighter.
+
+## Supported Software Versions
+
+This software is tested against the current Ruby 2.x series. It is
+unlikely to work without minor editing on 1.9.3, and you're on your own
+for anything earlier than 1.9.1.
+
+- See [.ruby-version][20] for the currently-supported Ruby versions.
+- See [Gemfile.lock][30] for a complete list of gems, including supported
+  versions, needed to build or run this project.
+
+## Installation and Setup
+
+Installing tweet\_compressor couldn't be easier. Just follow these two
+simple steps:
+
+1. `gem install tweet_compressor`
+2. There is no step two.
+
+## Usage
+
+    tweet_compressor <tweet>
+
+## Examples
+
+No screenshots here, just samples of what you can expect to see on
+standard output when you run the program.
+
+
+- Example of text that requires no compression.
+
+        $ tweet_compressor foo
+        Chars: 3, Compression: 0.0%
+
+        foo
+
+- Example of extremely heavy compression. Trims 196 characters about the
+  Gettysburg Address down to 137.
+
+        $ tweet_compressor 'Four score and seven years ago our fathers
+        brought forth on this continent a new nation, conceived in liberty,
+        and dedicated to the proposition that all men are created equal.
+        #speech #Lincoln'                                                            
+        Chars: 137, Compression: 28.65%
+
+        4 scr &7 yrs ago our fthrs brght frth on ths cntnt a new ntn,cncvd
+        in lbrty,& dctd to the prpstn tht al men are crtd eql.#speech
+        #Lincoln
+
+- Example of assumed compression from [Twitter's built-in URL
+  shortener.][10]
+
+        $ tweet_compressor 'http://tweet_compressor/knows/twitter/shortens/urls/to/20/characters'
+        Chars: 20, Compression: 70.59%
+
+        http://tweet_compressor/knows/twitter/shortens/urls/to/20/characters
+
+## Contributions Welcome
+
+This is an open-source project. Contributors are highly encouraged to
+open pull-requests on GitHub.
+
+----
+[Project Home Page](https://github.com/CodeGnome/tweet_compressor)
+
+[10]: https://support.twitter.com/entries/109623
+[20]: https://raw.github.com/CodeGnome/tweet_compressor/master/.ruby-version
+[30]: https://raw.github.com/CodeGnome/tweet_compressor/master/Gemfile.lock

data/Rakefile

@@ -0,0 +1,55 @@
+begin
+  require 'bundler/gem_tasks' if Dir.glob('*gemspec').any?
+  require 'bundler/setup'     if File.exists? 'Gemfile'
+rescue LoadError => bundler_missing
+  $stderr.puts bundler_missing
+end
+
+require 'rake'
+
+PROJECT_NAME = File.basename(Dir.pwd).sub /\.rb$/, ''
+
+desc 'Update exuberant-ctags'
+task :etags do
+  sh %{etags -R}
+end
+
+if Dir.exists? 'test'
+  require 'rake/testtask'
+
+  Rake::TestTask.new do |t|
+      t.test_files = FileList[ 'test*' ]
+  end
+  task :default => :test
+end
+
+if Dir.exists? 'spec'
+  require 'rspec/core/rake_task'
+  RSpec::Core::RakeTask.new(:spec)
+  task :default => :spec
+end
+
+desc 'Generate rdoc files'
+task :rdoc do
+  excludes = %w[AUTHORS LICENSE README* *gemspec]
+  system "rdoc #{excludes.map { |file| "-x #{file}" }.join ' '}"
+end
+
+task :rename_objects do
+  FileList['lib/**/**', 'README*', '.ruby-version', '.rvm'].each do |oldfile|
+    next if File.directory? oldfile
+    text = File.read(oldfile)
+
+    next unless text.match /(require|module|class).*foo/i
+    text.gsub!(/foo/i, PROJECT_NAME)
+    File.open(oldfile, 'w') { |f| f.puts text }
+  end
+end
+
+desc 'Rename lib files/objects'
+task :rename => :rename_objects do
+  libfiles = FileList['lib/**/**']
+  libfiles.gsub(/foo/, PROJECT_NAME).zip(libfiles).each do |f|
+    FileUtils.mv f[1], f[0] unless f.uniq.count == 1
+  end
+end

data/bin/tweet_compressor

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require_relative File.join '..', 'lib', 'tweet_compressor'
+
+unless ARGV.size == 1
+  puts "Usage: #{File.basename $0} <tweet>"
+  exit 1
+end
+
+tweet = TweetCompressor::Tweet.new ARGV.join ' '
+tweet.compress
+
+$stderr.puts "Chars: #{tweet.char_count}, Compression: #{tweet.compression_level}%"
+$stdout.puts ?\n, tweet.compressed

data/lib/tweet_compressor.rb

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+$: << File.join(Dir.pwd, 'lib')
+
+require 'tweet_compressor/compress'
+require 'tweet_compressor/tweet'
+require 'tweet_compressor/version'

data/lib/tweet_compressor/compress.rb

@@ -0,0 +1,187 @@
+# This module is a mixin for classes that want to use a very basic alphabetic
+# shorthand to reduce text size. The module performs in-place operations, and
+# expects to find a @compressed instance variable to work from.
+#
+# Example:
+# 
+#   include Compress
+#   @original = 'JavaScript'
+#   @compressed = @original.dup
+#   abbr
+#   # => "JS"
+#
+#
+module Compress
+  URL_HOLDER = '__PLACEHOLDER4URLS__'
+  URL_LENGTH = 20
+  URL_PATTERN  = %r{
+                     \b
+                     (
+                       (?: [a-z][\w-]+:
+                        (?: /{1,3} | [a-z0-9%] ) |
+                         www\d{0,3}[.] |
+                         [a-z0-9.\-]+[.][a-z]{2,4}/
+                       )
+                       (?:
+                        [^\s()<>]+ | \(([^\s()<>]+|(\([^\s()<>]+\)))*\)
+                       )+
+                       (?:
+                         \(([^\s()<>]+|(\([^\s()<>]+\)))*\) |
+                         [^\s`!()\[\]{};:'".,<>?«»“”‘’]
+                       )
+                     )
+                 }ix
+
+  # Calculate the current character count, taking the "virtual size" of
+  # Twitter-shortened URLs into account.
+  def char_count
+    real_url_chars = @urls.join.size
+    virt_url_chars = @urls.count * URL_LENGTH
+    @compressed.size - real_url_chars + virt_url_chars
+  end
+
+  private
+
+  # Special abbreviations to increase clarity.
+  #
+  # TODO: A YAML dictionary would be preferrable to case statements if the list
+  # grows to any significant length.
+  def abbr
+    @compressed = @compressed.split.map do |word|
+      case word.downcase
+      when 'and'        then '&'
+      when 'javascript' then 'JS'
+      when 'string'     then 'str'
+      when 'one'        then '1'
+      when 'two'        then '2'
+      when 'three'      then '3'
+      when 'four'       then '4'
+      when 'five'       then '5'
+      when 'six'        then '6'
+      when 'seven'      then '7'
+      when 'eight'      then '8'
+      when 'nine'       then '9'
+      when 'ten'        then '10'
+      when 'eleven'     then '11'
+      when 'twelve'     then '12'
+      when 'thirteen'   then '13'
+      when 'fourteen'   then '14'
+      when 'fifteen'    then '15'
+      when 'sixteen'    then '15'
+      when 'seventeen'  then '17'
+      when 'eighteen'   then '18'
+      when 'nineteen'   then '19'
+      when 'twenty'     then '20'
+      else word
+      end
+    end.join ' '
+    @compressed.gsub! /is (?:an?|the)/, '='
+    @compressed.gsub! /(in|with)? regards? (to)?/i, 're'
+    @compressed.gsub! /about|regarding|related( to)?|(in response to)/, 're'
+  end
+
+  # Remove apostrophes from contractions to save more space.
+  def apostrophes
+    @compressed.gsub! /n't/i, 'nt'
+  end
+
+  # Identify common contractions, taking a few pains to preserve capitalization
+  # of the initial letter.
+  def contractions
+    @compressed.gsub! /I would/i, %q{I'd}
+    @compressed.gsub! /i will(?!= ?not)/i, %q{I'll}
+    @compressed.gsub! /(i)t is/i, %q{\1t's}
+    @compressed.gsub! /(i)s not/i, %q{\1sn't}
+    @compressed.gsub! /(w)ill not/i, %q{\1on't}
+    @compressed.gsub! /(c)an ?not/i, %q{\1an't}
+    @compressed.gsub! /(d)o(es)? not/i, %q{\1o\2n't}
+    @compressed.gsub! /(s)hould not/i, %q{\1houldn't}
+    @compressed.gsub! /(m)ust not/i, %q{\1usn't}
+  end
+
+  # Fix common grammar mistakes that also save space.
+  def correct_grammar
+    @compressed.gsub! /s's/i, ?'
+  end
+
+  # Remove duplicate lowercase consonants. Assume duplicate capital letters
+  # like 'LLC' are intentional.
+  def dedupe_consonants
+    consonants = [*'a'..'z'].flatten.reject { |c| c =~ /[aeiou]/ }
+    regex = /(#{consonants})\1+/
+    @compressed = @compressed.split.map do |word|
+      next word unless word =~ regex
+      word.gsub! regex, $1.to_s
+    end.join ' '
+  end
+
+  # Remove duplicate punctuation characters. Make an exception for ellipses
+  # and dashes.
+  def dedupe_punct
+    regex = /([[:punct:]])\1+/
+    @compressed = @compressed.split.map do |word|
+      word.gsub! /\.{4,}/, '...'
+      word.gsub! /-{3,}/, '--'
+      next word if word.include? '...' or word.match /-{2,3}/
+      next word unless word =~ regex
+      word.gsub! regex, '\1'
+    end.join ' '
+  end
+
+  # Replace 'ing' with 'g'. Excludes short words like "ring" and "sing," and
+  # checks an exception list for special cases.
+  def ing
+    exceptions = %w[fling]
+    @compressed = @compressed.split.map do |word|
+      next word unless word.end_with? 'ing'
+      next word if word.start_with? '#'
+      next word if word.size <= 4
+      next word if exceptions.include? word
+      word.sub(/ing$/, 'g')
+    end.join ' '
+  end
+
+  # Remove lowercase vowels in longer words, unless it is the starting letter.
+  def remove_vowels
+    @compressed = @compressed.split.map do |word|
+      next word if word.start_with? '#'
+      word.size >= 4 ? word.gsub(/(?<!\A)[aeiou]/, '') : word
+    end.join ' '
+  end
+
+  # Remove spaces between punctuation marks and the following words.
+  def sentences
+    @compressed.gsub! /([[:punct:]])\s*(\S)/, '\1\2'
+  end
+
+  # Abbreviations common in texting, but with a higher cognitive load.
+  def texting
+    @compressed.gsub! /is (?:an?|the)/, '='
+    @compressed.gsub! /:.\)|\(.:/, ':)'
+    @compressed.gsub! /(in|with)? regards? (to)?/i, 're'
+    @compressed.gsub! /about|regarding|related( to)?|(in response to)/i, 're'
+    @compressed.gsub! /(RT @[^:\b]+):?/, '\1'
+    @compressed.gsub! /\bare\b/, 'r'
+    @compressed.gsub! /\bfor\b/, '4'
+    @compressed.gsub! /\bto/, '2'
+    @compressed.gsub! /why/, 'y'
+    @compressed.gsub! /you/, 'u'
+  end
+
+  # Regularize whitespace.
+  def whitespace
+    @compressed = @compressed.split.join ' '
+  end
+
+  # Temporarily remove URLs from the pattern space so that they don't get horked
+  # during other text transormations.
+  def url_preserve
+    @urls = @compressed.scan(/#{URL_PATTERN}/).flatten.compact
+    @urls.each { |url| @compressed.gsub! /#{url}/, URL_HOLDER }
+  end
+
+  # Return stored URLs to the pattern space.
+  def url_restore
+    @urls.each { |url| @compressed.sub! URL_HOLDER, url }
+  end
+end

data/lib/tweet_compressor/tweet.rb

@@ -0,0 +1,38 @@
+module TweetCompressor
+  class Tweet
+    MAX_LENGTH = 140
+    attr_reader :compressed, :original, :urls
+
+    def initialize tweet=''
+      @original, @compressed = tweet, tweet
+      @urls = []
+    end
+
+    # The workhorse method that calls each compression stage in turn as long as
+    # the tweet text remains larger than 140 characters.
+    def compress
+      # Always perform, in order to track URL shortening.
+      url_preserve
+
+      stages = %i[url_preserve whitespace correct_grammar contractions
+                  dedupe_punct abbr remove_vowels dedupe_consonants apostrophes
+                  sentences]
+      stages.each do |stage|
+        break if char_count <= MAX_LENGTH
+        self.send stage
+      end
+
+      # Must not be a stage, which may be bypassed.
+      url_restore
+
+      @compressed
+    end
+
+    def compression_level
+      (100 - ((char_count / @original.size.to_f) * 100)).round 2
+    end
+
+    private
+    include Compress
+  end
+end

data/lib/tweet_compressor/version.rb

@@ -0,0 +1,3 @@
+module TweetCompressor
+  VERSION = '0.8.2'
+end

data/spec/spec_helper.rb

@@ -0,0 +1,6 @@
+$: << Dir.pwd
+
+require 'simplecov'
+SimpleCov.start
+
+require 'tweet_compressor'