correct-horse-battery-staple 0.6.1

data.tar.gz.sig

@@ -0,0 +1 @@
-�<�	nF�"|�c�8���bd@�B@�o��h'{�N�� �@U����ł�u�<�@��C4���kڏ-�w)���
+��>��J�y���{�=�W2:ޣq��x����}(F�^f��j���b���~������e��#?��{��C~S��ա(
��v�+�כ�g���qE��u�E�v��~Y��pR��[

data/Gemfile

@@ -0,0 +1,53 @@
+source :rubygems
+
+# data formats
+gem "fastercsv", :platforms => [:mri_18, :jruby]
+gem "json"
+
+# performance
+# gem "memoizable"
+
+# external DBs
+gem "sqlite3", :platforms => [:mri]
+
+platform :mri do
+  gem "hiredis"
+  gem "redis", ">= 2.2.0" # , :require => ["redis", "redis/connection/hiredis"]
+end
+
+platform :jruby do
+  gem "redis", ">= 2.2.0"  
+end
+
+gem "tupalo-kdtree"
+
+# cmdline
+gem "commander"
+platform :jruby do
+  gem "ffi-ncurses"
+end
+
+gem "rdoc"
+
+group :test do
+  gem "rspec"
+end
+
+group :development do
+  # debugging
+  gem "pry"
+
+  # gem creation
+  gem "rubyforge"
+  gem "hoe"
+  gem "hoe-git"
+  gem "hoe-bundler"
+  gem "hoe-yard"
+  gem "hoe-gemspec"
+  gem "hoe-debugging"
+
+  gem "ruby-prof", :platforms => [:mri]
+
+  # CI
+  gem "tddium"
+end

data/Gemfile.lock

@@ -0,0 +1,109 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    coderay (0.9.8)
+    commander (4.0.6)
+      highline (~> 1.5.0)
+    diff-lcs (1.1.3)
+    fastercsv (1.5.4)
+    ffi (1.0.11-java)
+    ffi-locale (1.0.1)
+      ffi (>= 1.0.9)
+    ffi-ncurses (0.4.0)
+      ffi (>= 1.0.9)
+      ffi-locale (>= 1.0.0)
+    highline (1.5.2)
+    hiredis (0.4.4)
+    hoe (2.12.3)
+      rake (~> 0.8)
+    hoe-bundler (1.1.0)
+      hoe (>= 2.2.0)
+    hoe-debugging (1.0.2)
+      hoe (>= 2.2.0)
+    hoe-gemspec (1.0.0)
+      hoe (>= 2.2.0)
+    hoe-git (1.4.1)
+      hoe (>= 2.2.0)
+    hoe-yard (0.1.2)
+      yard (>= 0.2.3.1)
+    httparty (0.8.1)
+      multi_json
+      multi_xml
+    json (1.6.1)
+    json (1.6.1-java)
+    json_pure (1.6.3)
+    method_source (0.6.6)
+      ruby_parser (~> 2.0.5)
+    multi_json (1.0.4)
+    multi_xml (0.4.1)
+    pry (0.9.6.2)
+      coderay (~> 0.9.8)
+      method_source (~> 0.6.5)
+      ruby_parser (~> 2.0.5)
+      slop (~> 2.1.0)
+      spoon (~> 0.0.1)
+    pry (0.9.6.2-java)
+      coderay (~> 0.9.8)
+      method_source (~> 0.6.5)
+      ruby_parser (~> 2.0.5)
+      slop (~> 2.1.0)
+      spoon (~> 0.0.1)
+    rake (0.9.2)
+    rdoc (3.12)
+      json (~> 1.4)
+    redis (2.2.2)
+    rspec (2.5.0)
+      rspec-core (~> 2.5.0)
+      rspec-expectations (~> 2.5.0)
+      rspec-mocks (~> 2.5.0)
+    rspec-core (2.5.2)
+    rspec-expectations (2.5.0)
+      diff-lcs (~> 1.1.2)
+    rspec-mocks (2.5.0)
+    ruby-prof (0.10.8)
+    ruby_parser (2.0.6)
+      sexp_processor (~> 3.0)
+    rubyforge (2.0.4)
+      json_pure (>= 1.1.7)
+    sexp_processor (3.0.7)
+    slop (2.1.0)
+    spoon (0.0.1)
+    sqlite3 (1.3.5)
+    tddium (1.0.6)
+      bundler
+      highline
+      json
+      tddium_client (~> 0.1.1)
+      thor
+    tddium_client (0.1.3)
+      httparty (>= 0.8.1)
+      json
+    thor (0.14.6)
+    tupalo-kdtree (0.2.3)
+    yard (0.7.4)
+
+PLATFORMS
+  java
+  ruby
+
+DEPENDENCIES
+  commander
+  fastercsv
+  ffi-ncurses
+  hiredis
+  hoe
+  hoe-bundler
+  hoe-debugging
+  hoe-gemspec
+  hoe-git
+  hoe-yard
+  json
+  pry
+  rdoc
+  redis (>= 2.2.0)
+  rspec
+  ruby-prof
+  rubyforge
+  sqlite3
+  tddium
+  tupalo-kdtree

data/History.txt

@@ -0,0 +1,6 @@
+=== 1.0.0 / 2011-12-23
+
+* 1 major enhancement
+
+  * Birthday!
+

data/Manifest.txt

@@ -0,0 +1,57 @@
+Gemfile
+Gemfile.lock
+History.txt
+Manifest.txt
+README.txt
+Rakefile
+bin/chbs
+bin/chbs-mkpass
+lib/correct_horse_battery_staple.rb
+lib/correct_horse_battery_staple/assembler.rb
+lib/correct_horse_battery_staple/backend.rb
+lib/correct_horse_battery_staple/backend/isam_kd.rb
+lib/correct_horse_battery_staple/backend/redis.rb
+lib/correct_horse_battery_staple/backend/redis/d_range.rb
+lib/correct_horse_battery_staple/corpus.rb
+lib/correct_horse_battery_staple/corpus/base.rb
+lib/correct_horse_battery_staple/corpus/isam.rb
+lib/correct_horse_battery_staple/corpus/isam_kd.rb
+lib/correct_horse_battery_staple/corpus/redis.rb
+lib/correct_horse_battery_staple/corpus/redis2.rb
+lib/correct_horse_battery_staple/corpus/serialized.rb
+lib/correct_horse_battery_staple/corpus/sqlite.rb
+lib/correct_horse_battery_staple/generator.rb
+lib/correct_horse_battery_staple/memoize.rb
+lib/correct_horse_battery_staple/parser.rb
+lib/correct_horse_battery_staple/parser/base.rb
+lib/correct_horse_battery_staple/parser/regex.rb
+lib/correct_horse_battery_staple/range_parser.rb
+lib/correct_horse_battery_staple/statistical_array.rb
+lib/correct_horse_battery_staple/stats.rb
+lib/correct_horse_battery_staple/word.rb
+lib/correct_horse_battery_staple/writer.rb
+lib/correct_horse_battery_staple/writer/base.rb
+lib/correct_horse_battery_staple/writer/csv.rb
+lib/correct_horse_battery_staple/writer/file.rb
+lib/correct_horse_battery_staple/writer/isam.rb
+lib/correct_horse_battery_staple/writer/isam_kd.rb
+lib/correct_horse_battery_staple/writer/json.rb
+lib/correct_horse_battery_staple/writer/marshal.rb
+lib/correct_horse_battery_staple/writer/redis.rb
+lib/correct_horse_battery_staple/writer/sqlite.rb
+script/generate_all
+script/load_redis
+script/perftest
+spec/corpus/serialized_spec.rb
+spec/corpus_spec.rb
+spec/correct_horse_battery_staple_spec.rb
+spec/fixtures/100.json
+spec/fixtures/corpus1.csv
+spec/fixtures/corpus100.json
+spec/fixtures/wiktionary1000.htm
+spec/range_parser_spec.rb
+spec/spec_helper.rb
+spec/statistical_array_spec.rb
+spec/support/spec_pry.rb
+spec/word_spec.rb
+correct-horse-battery-staple.gemspec

data/README.txt

@@ -0,0 +1,115 @@
+= chbs
+
+* http://github.com/rsanders/correct-horse-battery-staple
+
+== DESCRIPTION:
+
+Generate a 4 word password from words of size 3-8 characters, with
+frequencies in the 30th-60th percentile. This range gives a nice set
+of uncommon but not completely alien words.
+
+    $ chbs generate --verbose -W 3..8 -P 30..60
+    Corpus size: 6396 candidate words of 33075 total
+    Entropy: 48 bits (2^48 = 281474976710656)
+    Years to guess at 1000 guesses/sec: 8926
+    magnate-thermal-sandbank-augur
+
+With the --verbose flag, the utility will calculate a time-to-guess
+based on a completely arbitrary 1000 guesses/sec.  If you'd like a
+more secure password, either relax the various filtering rules (-W and
+-P), add more words to the password, or use a larger corpus.
+
+By default we use the American TV Shows & Scripts corpus taken from
+Wiktionary.
+
+Others provided:
+
+* Project Gutenberg 2005 corpus taken from Wiktionary.
+* 1 of every 7 of the top 60000 lemmas from wordfrequency.info (6900
+  actual lemmas after processing)
+
+See http://xkcd.com/936/ for the genesis of the idea.
+
+Data sources:
+
+     http://en.wiktionary.org/wiki/Wiktionary:Frequency_lists
+     http://wordfrequency.info/
+
+== FEATURES/PROBLEMS:
+
+* Generates pretty decent XKCD-style passwords using pretty simple logic
+* Meant to be a proof-of-concept, and succeeds at that.
+
+Not so good:
+
+* Corpus loading is slow and memory-hungry
+* Needs a good corpus abstraction beyond serialized arrays/objects
+  loaded from CSV/JSON/Marshal
+* Should probably store default filter params per-corpus (e.g., 30-70
+  percentile works great for one corpus, badly for another)
+* Probably needs a ~/.correct-horse-battery-staple file to set defaults
+
+
+== SYNOPSIS:
+
+Command line usage, for a password of 4 words (default), each word of
+length between 3-8 letters, taken from the 30th through the 60th
+percentile range of the corpus sorted by word frequency from least to
+most frequent:
+
+    $ chbs generate --verbose -W 3..8 -P 30..60
+
+    Corpus size: 6396 candidate words of 33075 total
+    Entropy: 48 bits (2^48 = 281474976710656)
+    Years to guess at 1000 guesses/sec: 8926
+    magnate-thermal-sandbank-augur
+
+The 'chbs' command line program is the best reference for usage of the
+underlying library.  This would be a minimal version:
+
+    require 'correct_horse_battery_staple'
+    corpus    = CorrectHorseBatteryStaple.default_corpus
+    generator = CorrectHorseBatteryStaple::Generator.new(corpus)
+    puts generator.make(4)
+
+== REQUIREMENTS:
+
+* Ruby 1.8.7, 1.9.x, or (approximately) JRuby 1.5.x or later.
+
+== INSTALL:
+
+Just "gem install" and use the "chbs" wrapper program.
+
+== DEVELOPERS:
+
+After checking out the source, run:
+
+  $ rake newb
+
+This task will install any missing dependencies, run the tests/specs,
+and generate the RDoc.
+
+== LICENSE:
+
+(The MIT License)
+
+Copyright (c) 2011 Robert Sanders, opensource@esquimaux.otherinbox.com
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,47 @@
+# -*- ruby -*-
+
+require 'rubygems'
+require 'hoe'
+
+# Hoe.plugin :compiler
+# Hoe.plugin :gem_prelude_sucks
+# Hoe.plugin :inline
+# Hoe.plugin :racc
+
+Hoe.plugin :bundler
+Hoe.plugin :git
+Hoe.plugin :rubyforge
+Hoe.plugin :gemspec
+
+Hoe.spec 'correct-horse-battery-staple' do
+  developer('Robert Sanders', 'robert@curioussquid.com')
+  dependency 'commander', '>= 4.0'
+  dependency 'fastercsv', '>= 1.5.3'
+  dependency 'json', '>= 1.6.0'
+  dependency 'redis', '>= 2.2.2'
+  dependency 'hiredis', '>= 0.4.0'
+  dependency 'tupalo-kdtree', '>= 0.2.3'
+  dependency 'sqlite3', '>= 1.3.0'
+end
+
+namespace :chbs do
+  task :generate_corpus => "corpus/tvscripts.json"
+
+  file "corpus/tvscripts.json" do |task|
+    sh "./script/generate_all"
+  end
+  task :corpus => "corpus/tvscripts.json"
+
+  task :clean do
+    sh "rm -f corpus/*"
+  end
+end
+
+task :corpus => "chbs:corpus"
+
+["spec"].each do |task|
+  Rake::Task[task].prerequisites.unshift "chbs:corpus"
+end
+task :clean => "chbs:clean"
+
+# -*- mode: Ruby -*-

data/bin/chbs

@@ -0,0 +1,234 @@
+#!/usr/bin/env ruby
+
+require 'correct_horse_battery_staple'
+require 'correct_horse_battery_staple/assembler'
+require 'commander/import'
+require 'logger'
+
+program :version, "0.1.0"
+program :description, "Correct Horse Battery Staple - XKCD-style Passphrases"
+
+global_option '-D', '--debug', 'Display full protocol trace'
+global_option '-V', '--verbose', 'Shows request and response'
+
+logger = Logger.new(STDERR)
+
+default_command :generate
+
+command :repl do |c|
+  c.syntax = 'repl'
+  c.summary = 'Open Pry-based REPL, optionally with a loaded corpus'
+  c.option '-c CORPUSFILE', '--corpus CORPUSFILE', "File to use as the source corpus"
+  c.action do |args, options|
+    require 'pry'
+
+    if options.corpus
+      format = CorrectHorseBatteryStaple::Corpus.format_for(options.corpus)
+      corpus = CorrectHorseBatteryStaple.load_corpus(options.corpus, format)
+    end
+    
+    binding.pry
+  end
+end
+
+command :convert do |c|
+  c.syntax = 'convert <infile> [<outfile>]'
+  c.summary = 'Convert a corpus from one format to another'
+  c.option '-i IFORMAT', '--iformat IFORMAT', 'Set the output corpus format'
+  c.option '-o OFORMAT', '--oformat OFORMAT', 'Set the output corpus format'
+  c.option '-r', '--recalculate', 'Recalculate all statistics before outputting'
+  c.option '-S FIELD', '--sortby FIELD', 'Sort words by field'
+  c.option '-R', '--randomize', 'Randomize word order'
+  c.option '-l COUNT', '--limit COUNT', Integer, 'Set the maximum words processed'
+  c.option '-P', '--precache', 'Precache the source word list in memory'
+
+  c.action do |args, options|
+    options.default :limit => -1
+
+    infile = args[0]
+    outfile = args[1] && args[1] != "-" ?
+              (!args[1].include?(':') ? open(args[1], "w") : args[1]) :
+              STDOUT
+
+    options.iformat ||= CorrectHorseBatteryStaple::Corpus.format_for(infile)
+    unless options.oformat
+      if args[1]
+        options.oformat = CorrectHorseBatteryStaple::Corpus.format_for(args[1])
+      else
+        raise ArgumentError, "You must supply an output format via the file extension or -o option"
+      end
+    end
+
+    corpus = CorrectHorseBatteryStaple.load_corpus(args[0], options.iformat)
+    if options.precache
+      corpus.precache(16_000_000)
+    end
+
+    # CSV doesn't currently persist stats
+    corpus.recalculate if options.recalculate || options.iformat.to_s == "csv"
+
+    corpus.table.shuffle! if options.randomize
+    if options.limit > -1
+      corpus.table.slice!(options.limit..-1) if corpus.size > options.limit
+    end
+
+    with_tempfile(outfile) do |tempfile|
+      writer = CorrectHorseBatteryStaple::Writer.make_writer(tempfile, options.oformat)
+      writer.write_corpus(corpus)
+      writer.close
+    end
+
+  end
+end
+
+command :stats do |c|
+  c.syntax = 'stats <corpusfile>'
+  c.summary = 'Show statistics from a corpus'
+
+  c.action do |args,options|
+    corpus = CorrectHorseBatteryStaple::Corpus.read(args[0])
+
+    stats = corpus.stats
+    stats.to_hash.each do |key, val|
+      puts "#{key}: #{val}"
+    end
+  end
+end
+
+command :inspect do |c|
+  c.syntax = 'inspect <corpusfile>'
+  c.summary = 'Show implementation information for a corpus'
+
+  c.action do |args,options|
+    corpus = CorrectHorseBatteryStaple::Corpus.read(args[0])
+    puts corpus.inspect
+  end
+end
+
+command :generate do |c|
+  c.syntax = 'generate [<number of words>]'
+  c.summary = 'Generate a passphrase with a given number of words'
+  #c.description = ''
+  #c.example 'description', 'command example'
+  c.option '-f FORMAT', '--format FORMAT', 'Set the corpus format'
+  c.option '-c CORPUSFILE', '--corpus CORPUSFILE', "File to use as the source corpus"
+  c.option '-n COUNT', '--repeat COUNT', Integer, 'How many passwords to generate'
+  c.option '-P', '--precache', 'Precache the word list in memory'
+
+  # filters
+  c.option '-W MIN..MAX', '--wordsize MIN..MAX',
+           'Set the allowed word size in number of characters'
+  c.option '-P MIN..MAX', '--percentile MIN..MAX',
+           'Set the percentile range of the word frequency'
+
+  c.action do |args, options|
+    options.default :wordsize => "3..7", :repeat => 1,
+                    :corpus => CorrectHorseBatteryStaple::DEFAULT_CORPUS_NAME,
+                    :percentile => "30..80"
+
+    number_of_words = (args[0] || 4).to_i
+    count = options.repeat.to_i
+
+    corpus = CorrectHorseBatteryStaple.load_corpus(options.corpus, options.format)
+    if options.precache
+      corpus.precache(16_000_000)
+    end
+    puts "loaded corpus!" if options.debug
+    unfiltered_size = corpus.count
+
+    make_options = {}
+
+    f_wordsize = CorrectHorseBatteryStaple::RangeParser.new.parse options.wordsize
+    unless f_wordsize.begin <= 1 && f_wordsize.end >= 30
+      make_options[:word_length] = f_wordsize
+    end
+
+    f_percentile = CorrectHorseBatteryStaple::RangeParser.new.parse options.percentile
+    unless f_percentile.include? 0..100
+      make_options[:percentile] = f_percentile
+    end
+
+    if options.verbose
+      entropy = number_of_words * corpus.entropy_per_word.floor
+      guesses_per_sec = 1000.0
+      search_space = 2**entropy
+      years = search_space / guesses_per_sec / (365*24*3600)
+      puts "Corpus size: #{corpus.length} candidate words of #{unfiltered_size} total"
+      puts "Entropy: #{entropy} bits (2^#{entropy} = #{search_space})"
+      puts "Years to guess at #{guesses_per_sec.to_i} guesses/sec: #{years.round}"
+    end
+
+    generator = CorrectHorseBatteryStaple::Generator.new(corpus)
+
+    count.times do
+      puts generator.make(number_of_words, make_options)
+    end
+  end
+end
+
+command :list do |c|
+  c.syntax = 'list'
+  c.summary = 'Show a list of available corpus names'
+  c.option '-p', '--paths', 'Show paths'
+
+  c.action do |args, options|
+    list = CorrectHorseBatteryStaple.corpus_list(:with_paths => options.paths)
+
+    list.each_with_index do |item, i|
+      list[i] = "[#{item}]" if item.include? CorrectHorseBatteryStaple::DEFAULT_CORPUS_NAME
+    end
+    puts list.sort.join(options.paths ? "\n" : ', ')
+  end
+end
+
+command :mkcorpus do |c|
+  c.syntax = 'mkcorpus file [file, file, ...]'
+  c.summary = 'Generate a JSON or CSV corpus from an input file'
+  #c.description = ''
+  #c.example 'description', 'command example'
+  c.option '-o FILENAME', '--output FILENAME', 'Set the output filename or spec'
+  c.option '-I FORMAT', '--iformat FORMAT', 'Set the input format - "wiktionary" or "wordfrequency"'
+  c.option '-O FORMAT', '--oformat FORMAT', 'Set the output format - "marshal", "json", or "csv"'
+  c.option '-R', '--randomize', 'Randomize word order'
+  c.option '-l COUNT', '--limit COUNT', Integer, 'Set the maximum words processed'
+  c.action do |args, options|
+    options.default :iformat => "wiktionary", :limit => -1
+
+    assembler = CorrectHorseBatteryStaple::Assembler.new(
+      CorrectHorseBatteryStaple::Parser::Regex.new(options.iformat.to_sym))
+    assembler.read(args)
+    assembler.randomize if options.randomize
+    if options.limit > -1
+      assembler.limit(options.limit)
+    end
+    corpus = assembler.corpus
+
+    destination = options.output || "-"
+
+    options.oformat ||= CorrectHorseBatteryStaple::Corpus.format_for(destination)
+
+    with_tempfile(destination) do |tempname|
+      CorrectHorseBatteryStaple::Writer.write(corpus, tempname, options.oformat)
+    end
+  end
+end
+
+def with_tempfile(realname)
+  pathpart = realname.respond_to?(:path) ? realname.path : realname
+  tempname = nil
+  if (realname.is_a?(String) ||
+      (realname.is_a?(File) && File.exist?(realname.path))) &&
+      ! pathpart != '-' &&
+      ! pathpart.include?(':')
+    begin
+      tempname = File.join(File.dirname(pathpart), ".temp-#{File.basename(pathpart)}")
+      yield tempname
+      File.rename tempname, pathpart
+    rescue
+      File.delete(tempname) if tempname && File.exist?(tempname)
+      raise
+    end
+  else
+    yield realname
+  end
+end