oak 0.0.3 → 0.4.1

data/Gemfile

@@ -1,17 +1,5 @@
-source "http://rubygems.org"
-# Add dependencies required to use your gem here.
-# Example:
-#   gem "activesupport", ">= 2.3.5"
+source 'https://rubygems.org'
 
-# Add dependencies to develop your gem here.
-# Include everything needed to run rake, tests, features, etc.
-group :development do
-  gem "shoulda", ">= 0"
-  gem "bundler", "~> 1.0.0"
-  gem "jeweler", "~> 1.6.4"
-  gem "rcov", ">= 0"
-  gem 'rdoc', "~> 3.11"
-end
-
-gem 'thor', "~> 0.14.6"
+git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
 
+gemspec

data/LICENSE

@@ -0,0 +1,22 @@
+MIT License
+
+Copyright (c) 2016-2018 ProsperWorks, Inc.
+Copyright (c) 2018      Copper, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/Makefile

@@ -0,0 +1,113 @@
+#
+# Makefile
+#
+# "make test" tests the external behavior of cli utilities which are
+# not accessible to "rake test".
+#
+# author: jhw@prosperworks.com
+# incept: 2019-09-25
+#
+
+.SUFFIXES:
+SHELL         := bash
+DESTDIR       := build
+
+SOURCES       := $(shell find . -type f -name '*.rb')
+SOURCES       += $(shell find . -type f -name 'Gemfile*')
+SOURCES       += $(shell find . -type f -name 'Rakefile')
+SOURCES       += $(shell find . -type f -name '.rubocop.yml')
+
+.PHONY: all
+.PHONY: test
+all: test
+
+.PHONY: clean
+clean:
+	rm -rf $(DESTDIR)
+
+# "make test-rake" performs our traditional Ruby Minitest suite.
+#
+.PHONY: test-rake
+test test-rake: $(DESTDIR)/test-rake.ok
+$(DESTDIR)/test-rake.ok: $(SOURCES)
+	@mkdir -p $(dir $@)
+	bundle exec rake test
+	@touch $@
+
+# "make test-shell-basic" tests bin/oak.rb for loadability and basic behavior.
+#
+.PHONY: test-shell-basic
+test test-shell-basic: $(DESTDIR)/test-shell-basic.ok
+$(DESTDIR)/test-shell-basic.ok: $(SOURCES)
+	@mkdir -p $(dir $@)
+	@echo
+	@echo bin/oak is friendly and self-tests
+	@echo
+	bin/oak --self-test
+	rubydoctest bin/oak.rb
+	bin/oak --help
+	@echo
+	@echo bin/oak can decode its own output in some common cases.
+	@echo
+	set -o pipefail ; echo hello | bin/oak | bin/oak --mode decode-lines | diff <(echo hello) -
+	set -o pipefail ; cat Makefile | bin/oak --mode encode-file | bin/oak --mode decode-lines | diff Makefile -
+	set -o pipefail ; cat Makefile | bin/oak --mode encode-file | bin/oak --mode decode-file | diff Makefile -
+	set -o pipefail ; cat Makefile | bin/oak --mode encode-lines | bin/oak --mode decode-lines | diff -w Makefile -
+	set -o pipefail ; cat .git/index | bin/oak --mode encode-file | bin/oak --mode decode-file | diff .git/index -
+	@echo
+	@echo bin/oak has some kind of pointless crufty modes, too.
+	@echo
+	set -o pipefail ; echo | bin/oak --mode tests > /dev/null
+	set -o pipefail ; echo | bin/oak --mode crazy > /dev/null
+	@echo
+	@echo bin/oak passed all the basic tests.
+	@echo
+	@touch $@
+
+# This keychain is only used for tests in this Makefile.
+#
+export OAK_TEST_KEYS=foo,bar
+export OAK_TEST_KEY_foo=oak_3CNB_3725491808_52_RjFTQTMyX0qAlJNbIK4fwYY0kh5vNKF5mMpHK-ZBZkfFarRjVPxS_ok
+export OAK_TEST_KEY_bar=oak_3CNB_201101230_52_RjFTQTMyXxbYlRcFH8JgiFNZMbnlFTAfUyvJCnXgCESpBmav_Etp_ok
+
+# "make test-shell-encryption" tests the encryption features in bin/oak.
+#
+.PHONY: test-shell-encryption
+test test-shell-encryption: $(DESTDIR)/test-shell-encryption.ok
+$(DESTDIR)/test-shell-encryption.ok: $(SOURCES)
+	@mkdir -p $(dir $@)
+	set -o pipefail ; bin/oak --key-generate | grep '^oak_3[-_0-9a-zA-Z]*_ok$$'
+	set -o pipefail ; cat Makefile | bin/oak --mode encode-lines --key-chain OAK_TEST --key foo | bin/oak --mode decode-lines --key-chain OAK_TEST | diff -w Makefile -
+	set -o pipefail ; cat Makefile | bin/oak --mode encode-file --key-chain OAK_TEST --key bar | bin/oak --mode decode-file --key-chain OAK_TEST | diff Makefile -
+	set -o pipefail ; bin/oak --key-check | grep 'no --key-chain specified'
+	set -o pipefail ; bin/oak --key-check --key-chain BOGUS_KEY_CHAIN_TEST | grep 'BOGUS_KEY_CHAIN_TEST: no keys found'
+	set -o pipefail ; bin/oak --key-check --key-chain OAK_TEST | grep 'OAK_TEST: found keys: foo bar'
+	@echo
+	@echo We can recognize which key encrypted which OAK string:
+	@echo
+	echo Hello | bin/oak --mode encode-file --key-chain OAK_TEST --key foo | grep '^oak_4foo' > /dev/null
+	echo Hello | bin/oak --mode encode-file --key-chain OAK_TEST --key bar | grep '^oak_4bar' > /dev/null
+	@echo
+	@echo We can recode one from one key to another
+	@echo
+	echo Hello | bin/oak --mode encode-file --key-chain OAK_TEST --key foo | grep '^oak_4foo' > /dev/null
+	echo Hello | bin/oak --mode encode-file --key-chain OAK_TEST --key foo | bin/oak --mode recode-file --key-chain OAK_TEST --key bar | grep '^oak_4bar' > /dev/null
+	@echo
+	@echo Both are decodeable as the original source:
+	@echo
+	echo Hello | bin/oak --mode encode-file --key-chain OAK_TEST --key foo | bin/oak --key-chain OAK_TEST --mode decode-file | diff <(echo Hello) - > /dev/null
+	echo Hello | bin/oak --mode encode-file --key-chain OAK_TEST --key foo | bin/oak --mode recode-file --key-chain OAK_TEST --key bar | bin/oak --key-chain OAK_TEST --mode decode-file | diff <(echo Hello) - > /dev/null
+	@echo
+	@echo bin/oak passed all the encryption tests.
+	@echo
+	@touch $@
+
+# "make test-rubocop" performs our rubocop checks.
+#
+.PHONY: test-rubocop
+test test-rubocop: $(DESTDIR)/test-rubocop.ok
+$(DESTDIR)/test-rubocop.ok: $(SOURCES)
+	@mkdir -p $(dir $@)
+	bundle exec rubocop --version
+	bundle exec rubocop --display-cop-names --display-style-guide
+	@touch $@

data/README.md

@@ -1,34 +1,174 @@
-## oak
+# oak
 
-Oak is a very simple tool that assists making rails application Open Source ready.
+OAK is a serialization format for Ruby primitives which supports
+performance experimentation via a limited suite of built-in
+compression, checksumming, and encryption options.
 
-**ONLY WORKS ON MAC OS X AND LINUX/BSD**
+OAK supports built-in switchability between common algorithms:
 
-When pushlishing your rails app to a public repository, it is often neccessary to find a way protecting you secret token and other private information. 
+  - Checksumming with ZLIB crc32(), SHA1, or none.
+  - Compression with LZ4, ZLIB, BZIP2k, LZMA, or none.
+  - ASCII armor with base64, or none.
 
-Oak helps you on these tasks by running 
+OAK also supports optional AES-256-GCM encryption with full 96-bit
+random IVs and 16 byte auth tags.
 
-    oak setup
+OAK is also a compact serialization format for primitive Ruby objects
+like String, Symbol, Integer, Float, Hash, and Array.  OAK
+serialization supports cyclical structures and distinguishes between
+String and Symbol, but does not support user-defined types.
 
-Then your private information will be stored in a file called 'config.yml', and automaticly ignored by git master branch. And on another branch named 'deploy' (which won't be pushed to the public repository) the config.yml file is included.
-So you can modify your apps on master branch as usual, and call
+In ProsperWorks/ALI, we use OAK at Copper, OAK has found use cases in:
 
-    git push <production_repo> deploy:master
-    
-to push your local deploy branch to your production repository's master branch.
+  - Volatile Redis cache entries, where dynamic control of time-space
+    tradeoffs shaves down hosting costs.
+  - Durable cold archives of larger user data.
+  - Encrypting runtime secrets.
 
-## Contributing to oak
- 
-* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet
-* Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it
-* Fork the project
-* Start a feature/bugfix branch
-* Commit and push until you are happy with your contribution
-* Make sure to add tests for it. This is important so I don't break it in a future version unintentionally.
-* Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
+OAK is not human-readable.  It is intended to be an interchange or
+archive format, not as a human interface.
 
-## Copyright
+Consider OAK if you are dealing with content at scale and are not
+quite happy with JSON, YAML, XML, or Ruby's Marshal format.  Disregard
+OAK your needs are all met by JSON.
 
-Copyright (c) 2011 Huang Wei. See LICENSE.txt for
-further details.
+## Using in Ruby
 
+```
+2.1.6 :001 > require 'oak'
+ => true
+2.1.6 :002 > h = { 'one' => 1, 'array' => [ true, false ] }
+ => {"one"=>1, "array"=>[true, false]}
+2.1.6 :003 > OAK.encode(h)
+ => "oak_3CNB_2774455364_51_RjdIMl8xXzJfM180U1UzX29uZUkxU1U1X2FycmF5QTJfNV82dGY_ok"
+2.1.6 :004 > OAK.encode(h,format: :none)
+ => "oak_3CNN_2774455364_38_F7H2_1_2_3_4SU3_oneI1SU5_arrayA2_5_6tf_ok"
+2.1.6 :005 > OAK.encode(h, compression: :bzip2, force: true)
+ => "oak_3CBB_2774455364_106_QlpoOTFBWSZTWag9FGUAAAaPgD-AIWAKAKMBlCAgADFGjIGjTI0Ip-lPRGynomJ-qPMBxIQDw5vmY9SVFxhFj7ZLMSPxdyRThQkKg9FGUA_ok"
+2.1.6 :006 > OAK.encode(h, compression: :bzip2)
+ => "oak_3CNB_2774455364_51_RjdIMl8xXzJfM180U1UzX29uZUkxU1U1X2FycmF5QTJfNV82dGY_ok"
+2.1.6 :007 > OAK.decode('oak_3CNB_2774455364_51_RjdIMl8xXzJfM180U1UzX29uZUkxU1U1X2FycmF5QTJfNV82dGY_ok')
+ => {"one"=>1, "array"=>[true, false]}
+```
+
+## Using in Ruby for Encryption
+
+```
+2.1.6 :001 > require 'oak'
+ => true
+2.1.6 :002 > key_chain = OAK::KeyChain.new({ 'a' => OAK::Key.new(OAK.random_key), 'b' => OAK::Key.new(OAK.random_key) })
+ => #<OAK::KeyChain:0x007faa700b8458 @keys={"a"=>#<OAK::Key:0x007faa700b89a8 @key=ELIDED>, "b"=>#<OAK::Key:0x007faa700b86d8 @key=ELIDED>}>
+2.1.6 :003 > OAK.encode("Hello, World!",key_chain: key_chain, key: 'a')
+ => "oak_4a_B82_4iauuiFq7XQvcDCwmmXhDxL_Wp3_T765qM-S094uNJ4xtj_DxQuXGSMmqlcxPXz-_cYZdE6bFbEAQCHpiQ_ok"
+2.1.6 :004 > OAK.encode("Hello, World!",key_chain: key_chain, key: 'a')
+ => "oak_4a_B82_YeeAEOLPlGchlDkoSnif7G38uJAPaNQ2ozsx6Mcb7PybpsL-ljVmGa5sRbgaqFw4R5iOXNw_sOesolTB4g_ok"
+2.1.6 :005 > OAK.encode("Hello, World!",key_chain: key_chain, key: 'b')
+ => "oak_4b_B82_HA1O3v5UrpV81UC0fAaUQ_8tsLUxbMG6bMupcQaZNKMU3XL3Tz9zj8TaVb4nvv3s0UhxFg3q9lmFIplvnQ_ok"
+2.1.6 :006 > OAK.decode("oak_4b_B82_HA1O3v5UrpV81UC0fAaUQ_8tsLUxbMG6bMupcQaZNKMU3XL3Tz9zj8TaVb4nvv3s0UhxFg3q9lmFIplvnQ_ok",key_chain: key_chain)
+ => "Hello, World!
+2.1.6 :007 > OAK.decode("oak_4b_B82_HA1O3v5UrpV81UC0fAaUQ_8tsLUxbMG6bMupcQaZNKMU3XL3Tz9zj8TaVb4nvv3s0UhxFg3q9lmFIplvnQ_ok")
+OAK::CantTouchThisStringError: key b but no key_chain
+        from ...
+```
+
+## Using in Shell
+
+Non-encrypted OAK default to OAK3.  Unencrypted OAK4 is available
+optionally.  They are generally the same size and equivalent:
+unencrypted OAK4 is generally only used for debugging and getting a
+peek at what happens "under the encryption".
+
+```
+$ echo hello | bin/oak --mode encode-file
+oak_3CNB_911092726_16_RjFTVTZfaGVsbG8K_ok
+$ echo oak_3CNB_911092726_16_RjFTVTZfaGVsbG8K_ok | bin/oak --mode decode-file
+hello
+$ echo hello | bin/oak --mode encode-file --compression lz4 --force true
+oak_3C4B_911092726_19_DMBGMVNVNl9oZWxsbwo_ok
+$ echo oak_3C4B_911092726_19_DMBGMVNVNl9oZWxsbwo_ok | bin/oak --mode decode-file
+hello
+$ echo hello | bin/oak.rb --mode encode-lines --redundancy none --format none
+oak_3NNN_0_11_F1SU5_hello_ok
+$ echo hello | bin/oak.rb --mode encode-lines --redundancy none --format none --force-oak-4
+oak_4_N15_NN0_F1SU5_hello_ok
+```
+
+## Using in Shell for Encryption
+
+Encryption is supported by OAK4.
+
+```
+$ bin/oak.rb --key-generate
+oak_3CNB_2975186575_52_RjFTQTMyX00du8vD8WAikhLNgdnaOYtQV6uqyNqRz6modiEcJHOl_ok
+$ bin/oak.rb --key-generate
+oak_3CNB_1324948677_52_RjFTQTMyXytCueDDTpEOusKkPMANgaA9zsJuvOend5DCIJWwJdjC_ok
+$ export OAK_TEST_KEYS=foo,bar
+$ export OAK_TEST_KEY_foo=oak_3CNB_2975186575_52_RjFTQTMyX00du8vD8WAikhLNgdnaOYtQV6uqyNqRz6modiEcJHOl_ok
+$ export OAK_TEST_KEY_bar=oak_3CNB_1324948677_52_RjFTQTMyXytCueDDTpEOusKkPMANgaA9zsJuvOend5DCIJWwJdjC_ok
+$ echo hello | bin/oak.rb --mode encode-lines --redundancy none --key-chain OAK_TEST --key foo
+oak_4foo_B58_PhG1qWHfosOOWDgqMhVoZlEn6F16XC6KuL_1zN1aLWMmcZZgJ2Dz5XR-ag_ok
+$ echo hello | bin/oak.rb --mode encode-lines --redundancy none --key-chain OAK_TEST --key foo
+oak_4foo_B58_ms11iWDHrmwFJwGpNEsWMIXYfapO96e7yvfk5r8G-F1gRzt62FS_JFQbvw_ok
+$ echo hello | bin/oak.rb --mode encode-lines --redundancy none --key-chain OAK_TEST --key bar
+oak_4bar_B58_kV6FIE30v6xgdKwyzdmpxVzNCU2eWjt7ZiZTWUHsQxXG3cC8u0-VoE0hmQ_ok
+$ echo oak_4foo_B58_PhG1qWHfosOOWDgqMhVoZlEn6F16XC6KuL_1zN1aLWMmcZZgJ2Dz5XR-ag_ok | bin/oak.rb --mode decode-lines --key-chain OAK_TEST
+hello
+$ echo oak_4foo_B58_ms11iWDHrmwFJwGpNEsWMIXYfapO96e7yvfk5r8G-F1gRzt62FS_JFQbvw_ok | bin/oak.rb --mode decode-lines --key-chain OAK_TEST
+hello
+$ echo oak_4bar_B58_kV6FIE30v6xgdKwyzdmpxVzNCU2eWjt7ZiZTWUHsQxXG3cC8u0-VoE0hmQ_ok | bin/oak.rb --mode decode-lines --key-chain OAK_TEST
+hello
+```
+OAK4 supports one and only one encryption algorithm and mode of
+operation:
+
+- AES-256-GCM
+  - 128 bits of security
+  - 256-bit keys      (32 bytes)
+  -  96-bit IVs       (12 bytes)
+  - 128-bit auth_tags (16 bytes)
+- Random IV ("Initialization Vector") used for each encryption operation.
+- All headers are authenticated.
+- All headers which are not required for decryption are encrypted.
+
+This is the only encryption option supported by OAK4.
+
+Because the GCM ([Galois/Counter Mode
+](https://en.wikipedia.org/wiki/Galois%2FCounter_Mode)) mode of
+operation is an [authenticated
+encryption](https://en.wikipedia.org/wiki/Authenticated_encryption),
+use of `--redundancy none` with encrypted OAK strings is recommended.
+The authentication of GCM is more than adequate to detect accidental
+transmission errors.  This recommendation may become the default in a
+future version.
+
+
+## Further Reading
+
+For more details.
+
+- [Changelog](CHANGELOG.md)
+- [Design Desiderata](DESIDERATA.md)
+
+## TODO: packaging
+
+- import design docs here
+  - https://docs.google.com/document/d/10HVWuQzCw1Whc-czDChwsWPEZRLfyPS7F-dkjHWsIs4
+  - https://docs.google.com/document/d/1J7GBEJUPI3UeftJ4C-w3pbBUzk1pU0Sr6zcXufEi7NI
+  - https://docs.google.com/document/d/1SeOO18uqdDtHuB8tZ4-_2sql0Yiaco5J1PWdiu6gmAY
+- edit comments down
+- rdoc
+
+## Possible Future Directions for the Format
+
+- Float representation.
+  - Manifest precision?
+  - Limited precision?
+  - But somehow do better than just `Float#to_s`!
+- Streamability and embedability.
+  - Support encoders and decoders which have only fixed-size buffers.
+- Portability.
+  - Was support for Symbols distinct from Strings too Ruby-esque?
+  - Native implementation.
+- Error-correction coding.
+  - There seem to be little or no "standard" algorithms out there, at
+    least not as used by the Ruby community.

data/Rakefile

@@ -1,53 +1,12 @@
-# encoding: utf-8
-
 require 'rubygems'
-require 'bundler'
-begin
-  Bundler.setup(:default, :development)
-rescue Bundler::BundlerError => e
-  $stderr.puts e.message
-  $stderr.puts "Run `bundle install` to install missing gems"
-  exit e.status_code
-end
-require 'rake'
-
-require 'jeweler'
-Jeweler::Tasks.new do |gem|
-  # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
-  gem.name = "oak"
-  gem.homepage = "http://github.com/imonyse/oak"
-  gem.license = "MIT"
-  gem.summary = %Q{Oak is a tool that helps making your newly created rails app open source ready.}
-  gem.description = %Q{When pushing your rails app to a public repository, it is often neccessary to find a way protecting you secret token and other private information. Oak helps you on these tasks and assumes you use git as the version control system}
-  gem.email = "imonyse@gmail.com"
-  gem.authors = ["Huang Wei"]
-  # dependencies defined in Gemfile
-end
-Jeweler::RubygemsDotOrgTasks.new
-
+require 'bundler/setup'
+require 'bundler/gem_tasks'
 require 'rake/testtask'
-Rake::TestTask.new(:test) do |test|
-  test.libs << 'lib' << 'test'
-  test.pattern = 'test/**/test_*.rb'
-  test.verbose = true
-end
 
-require 'rcov/rcovtask'
-Rcov::RcovTask.new do |test|
-  test.libs << 'test'
-  test.pattern = 'test/**/test_*.rb'
-  test.verbose = true
-  test.rcov_opts << '--exclude "gems/*"'
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'test'
+  t.libs << 'lib'
+  t.test_files = FileList['test/**/*_test.rb']
 end
 
 task :default => :test
-
-require 'rdoc/task'
-RDoc::Task.new do |rdoc|
-  version = File.exist?('VERSION') ? File.read('VERSION') : ""
-
-  rdoc.rdoc_dir = 'rdoc'
-  rdoc.title = "oak #{version}"
-  rdoc.rdoc_files.include('README*')
-  rdoc.rdoc_files.include('lib/**/*.rb')
-end

data/bin/oak

@@ -1,6 +1,245 @@
 #!/usr/bin/env ruby
+#
+# oak.rb: cli driver for encoding strings in the OAK format.
+#
+# author: jhw@prosperworks.com
+# incept: 2016-03-05
+#
 
-$LOAD_PATH.unshift File.join(File.dirname(__FILE__), '..', 'lib')
-require 'oak'
+require_relative '../lib/oak.rb'
+require          'optimist'
 
-Oak.start
+OLD_ARGV = ARGV.dup            # ARGV is consumed by Optimist but we use later.
+OPTS     = Optimist.options do
+  banner "#{$0} cli driver for OAK"
+  banner <<-OPTIMIST_EXAMPLES
+Examples:
+  $ echo hello | bin/oak.rb
+  oak_3CNB_1944283675_15_RjFTVTVfaGVsbG8_ok
+  $ (echo hello ; echo world) | bin/oak.rb
+  oak_3CNB_1944283675_15_RjFTVTVfaGVsbG8_ok
+  oak_3CNB_2139413982_15_RjFTVTVfd29ybGQ_ok
+  $ (echo hello ; echo world) | bin/oak.rb --compression zlib --force
+  oak_3CZB_1944283675_26_eJxzMwwONY3PSM3JyQcAFF4DyA_ok
+  oak_3CZB_2139413982_26_eJxzMwwONY0vzy_KSQEAFNgD3A_ok
+  $ (echo hello ; echo world) | bin/oak.rb --format none
+  oak_3CNN_1944283675_11_F1SU5_hello_ok
+  oak_3CNN_2139413982_11_F1SU5_world_ok
+  $ (echo hello ; echo world) | bin/oak.rb | bin/oak.rb --mode decode-lines
+  hello
+  world
+OPTIMIST_EXAMPLES
+  banner "Options:"
+  version "#{$0} #{OAK::VERSION}"
+  opt :redundancy,   'redundancy',                   :default => 'crc32'
+  opt :format,       'format',                       :default => 'base64'
+  opt :compression,  'compression',                  :default => 'none'
+  opt :force,        'compress even if bigger',      :default => false
+  opt :mode,         'mode',                         :default => 'encode-lines'
+  opt :key_chain,    'key chain env name',           :type    => :string
+  opt :key,          'encrypt key name',             :type    => :string
+  opt :key_check,    'check available keys',         :default => false
+  opt :key_generate, 'generate new key',             :default => false
+  opt :force_oak_4,  'force OAK_4 even unencrypted', :default => false
+  opt :eigen,        'calc eigenratio',              :type    => :int
+  opt :self_test,    'self-test only',               :default => false
+  opt :help,         'show this help'
+end
+Optimist::die :eigen, "must be non-negative" if OPTS[:eigen] && OPTS[:eigen] < 0
+
+oak_opts               = {}
+oak_opts[:redundancy]  = OPTS[:redundancy]
+oak_opts[:compression] = OPTS[:compression]
+oak_opts[:force]       = OPTS[:force]
+oak_opts[:format]      = OPTS[:format]
+oak_opts[:key_chain]   = OAK.parse_env_chain(ENV,OPTS[:key_chain])
+oak_opts[:key]         = OPTS[:key]
+oak_opts[:force_oak_4] = OPTS[:force_oak_4]
+
+if !OAK::REDUNDANCY_2_CODE.keys.include?(oak_opts[:redundancy])
+  Optimist::die :redundancy, "bogus #{OPTS[:redundancy]}"
+end
+if !OAK::COMPRESSION_2_CODE.keys.include?(oak_opts[:compression])
+  Optimist::die :compression, "bogus #{OPTS[:compression]}"
+end
+cool_formats = OAK::FORMAT_2_CODE.keys
+if !cool_formats.include?(oak_opts[:format])
+  Optimist::die :format, "bogus #{OPTS[:format]} not in #{cool_formats}"
+end
+
+=begin
+
+doctest: simple transcoding
+>> OAK::decode(OAK::encode([1,"2",3.000001]))
+=> [1,"2",3.000001]
+>> OAK::decode(OAK::encode({foo: "bar"}))
+=> {foo: "bar"}
+>> OAK::decode(OAK::encode({foo: :bar}))
+=> {foo: :bar}
+>> OAK::decode(OAK::encode("Hello, World!"))
+=> "Hello, World!"
+>> OAK::decode(OAK::encode("Hello, World!", format: :none, redundancy: :none))
+=> "Hello, World!"
+
+doctest: stability of encoding
+>> OAK::decode("oak_3NNB_0_30_RjNIMV8xXzJZQTNfZm9vU1UzX2Jhcg_ok")
+=> {:foo=>"bar"}
+>> OAK::encode(1, format: :base64, redundancy: :none)
+=> "oak_3NNB_0_6_RjFJMQ_ok"
+>> OAK::encode(1, format: :base64, redundancy: :crc32)
+=> "oak_3CNB_3405226796_6_RjFJMQ_ok"
+>> OAK::encode(1, format: :none, redundancy: :crc32)
+=> "oak_3CNN_3405226796_4_F1I1_ok"
+>> hello_utf8 = "Hello, World!".force_encoding('UTF-8')
+=> "Hello, World!"
+>> OAK::encode(hello_utf8, format: :base64, redundancy: :none)
+=> "oak_3NNB_0_27_RjFTVTEzX0hlbGxvLCBXb3JsZCE_ok"
+>> OAK::encode(hello_utf8, format: :none,   redundancy: :crc32)
+=> "oak_3CNN_2351984628_20_F1SU13_Hello, World!_ok"
+
+Note above I used force_encoding('UTF-8') after discovering that with
+Ruby 2.1.6 on Mac I get Encoding.default_encoding is UTF-8, but with
+Ruby 2.1.6 on Linux I get Encoding.default_encoding is US-ASCII!
+
+=end
+
+if __FILE__ == $0
+  if OPTS[:self_test]
+    require 'rubydoctest'
+    exit RubyDocTest::Runner.new(File.read(__FILE__), __FILE__).run ? 0 : 1
+  end
+  if OPTS[:key_check]
+    if !OPTS[:key_chain]
+      puts "no --key-chain specified"
+    else
+      keys = oak_opts[:key_chain].keys.keys
+      if 0 == keys.size
+        puts "#{OPTS[:key_chain]}: no keys found"
+      else
+        puts "#{OPTS[:key_chain]}: found keys: #{keys.join(' ')}"
+      end
+    end
+  end
+  if OPTS[:key_generate]
+    STDOUT.puts OAK.encode(OAK.random_key)
+    exit 0
+  end
+  if !$stdin.tty?
+    if OPTS[:eigen]
+      prev = STDIN.read
+      puts "input: %d" % prev.size
+      OPTS[:eigen].times do |i|
+        oak   = OAK.encode(prev,oak_opts)
+        psize = prev.size
+        wsize = oak.size
+        ratio = 1.0 * wsize / psize
+        puts "  iter %3d: %4d => %4d ratio %.2f" % [i,psize,wsize,ratio]
+        prev  = oak
+      end
+      exit 0
+    end
+    unhappiness = 0
+    case OPTS[:mode]
+    when 'cat'
+      ARGF.each_line.map(&:strip).each do |line|
+        puts line
+      end
+    when 'encode-lines'
+      ARGF.each_line.map(&:strip).each do |line|
+        puts OAK.encode(line,oak_opts)
+      end
+    when 'decode-lines'
+      ARGF.each_line.map(&:strip).each do |line|
+        puts OAK.decode(line,oak_opts)
+      end
+    when 'encode-file'
+      puts OAK.encode(STDIN.read,oak_opts)
+    when 'decode-file'
+      STDOUT.write OAK.decode(STDIN.read.strip,oak_opts)
+    when 'recode-file'
+      puts OAK.encode(OAK.decode(STDIN.read,oak_opts),oak_opts)
+    when 'crazy'
+      #
+      # --mode crazy prints out a sample of OAK strings for various
+      # challenging cases.
+      #
+      cycle_a    = ['cycle_a','TBD']
+      cycle_b    = ['cycle_b',cycle_a]
+      cycle_a[1] = cycle_b
+      dag_c      = ['dag_c']
+      dag_b      = ['dag_b',dag_c]
+      dag_a      = ['dag_a',dag_b,dag_c]
+      [
+        'hello',
+        ['hello'] + ['hello',:hello] * 2,
+        {1=>'a','b'=>2,[]=>3,''=>4,{}=>5,nil=>6},
+        ['x','x','x','x','x','x','x','x','x','x','x','x','x'],
+        ['x'] * 13,
+        cycle_a,
+        dag_a,
+        [1,-123,0.12,-0.123,Float::NAN,-Float::INFINITY,3.14159265358979],
+      ].each do |obj|
+        oak = OAK.encode(
+          obj,
+          redundancy:  :crc32,
+          format:      :none,
+          compression: :none,
+        )
+        puts ""
+        puts "obj:   #{obj}"
+        puts "  oak: #{oak}"
+        begin
+          dec = OAK.decode(oak,oak_opts)
+          if dec != obj
+            if !dec.is_a?(Float) && !enc.is_a?(Float) && !dec.nan? && !enc.nan?
+              unhappiness += 1
+              puts "  BAD: #{dec}"
+            end
+          end
+        rescue OAK::CantTouchThisStringError => ex
+          puts "  BAD: #{ex.message}: #{ex.backtrace_locations[0]}"
+          unhappiness += 1
+        end
+      end
+    when 'tests'
+      [
+        [1,2,3],
+        {:foo=>'foo','foo'=>['x']*10},
+        -1,
+        Float::NAN,
+        nil,
+      ].each do |obj|
+        puts "    #{obj} => ["
+        key_chain = OAK::KeyChain.new(
+          { 'l0ng3r' => OAK::Key.new('xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx') }
+        )
+        [
+          {redundancy: :none, format: :none,  compression: :none             },
+          {redundancy: :none, format: :base64,compression: :lz4,  force: true},
+          {redundancy: :crc32,format: :base64,compression: :zlib, force: true},
+          {redundancy: :crc32,format: :base64,compression: :bzip2,force: true},
+          {redundancy: :sha1, format: :base64,compression: :lzma, force: true},
+          {key_chain: key_chain,force_oak_4: true,format: :none,             },
+          {key_chain: key_chain,force_oak_4: true,                           },
+          {key_chain: key_chain,key: 'l0ng3r',                               },
+        ].each do |opts|
+          oak = OAK.encode(obj,opts)
+          puts "      '#{oak}',"
+          dec = OAK.decode(oak,opts)
+          if dec != obj
+            if !dec.is_a?(Float) && !enc.is_a?(Float) && !dec.nan? && !enc.nan?
+              unhappiness += 1
+            end
+          end
+        end
+        puts "    ],"
+      end
+    else
+      Optimist::die :mode, "bogus mode #{OPTS[:mode]}"
+    end
+    if unhappiness > 0
+      puts "unhappiness: #{unhappiness}"
+    end
+    exit unhappiness
+  end
+end