melissadata 0.0.1 → 0.1.0

data/Gemfile

@@ -1,5 +1,7 @@
 source "http://rubygems.org"
 
+gem 'vagrant', :git => 'https://github.com/mitchellh/vagrant.git'
+
 gemspec
 
 # #############################################################################

data/Gemfile.lock

@@ -1,3 +1,17 @@
+GIT
+  remote: https://github.com/mitchellh/vagrant.git
+  revision: a42ab2ce208a39df783c5942c50f46883cbf2f4c
+  specs:
+    vagrant (0.8.3.dev)
+      archive-tar-minitar (= 0.5.2)
+      erubis (~> 2.7.0)
+      i18n (~> 0.5.0)
+      json (~> 1.5.1)
+      net-scp (~> 1.0.4)
+      net-ssh (~> 2.1.4)
+      thor (~> 0.14.6)
+      virtualbox (~> 0.9.1)
+
 PATH
   remote: .
   specs:
@@ -15,29 +29,34 @@ GEM
   remote: http://rubygems.org/
   specs:
     activesupport (3.0.9)
+    archive-tar-minitar (0.5.2)
     awesome_print (0.4.0)
     cloudfiles (1.4.17)
       mime-types (>= 1.16)
     columnize (0.3.4)
     diff-lcs (1.1.2)
+    erubis (2.7.0)
+    ffi (1.0.9)
     growl (1.0.3)
     guard (0.5.1)
       thor (~> 0.14.6)
-    guard-rspec (0.4.0)
+    guard-rspec (0.4.1)
       guard (>= 0.4.0)
-    i18n (0.6.0)
+    i18n (0.5.0)
     iobuffer (1.0.0)
-    linecache (0.46)
-      rbx-require-relative (> 0.0.4)
+    json (1.5.3)
+    linecache19 (0.5.12)
+      ruby_core_source (>= 0.1.4)
     mime-types (1.16)
-    msgpack (0.4.5)
+    msgpack (0.4.6)
     msgpack-rpc (0.4.5)
       msgpack (>= 0.4.4)
       rev (>= 0.3.0)
     multi_json (1.0.3)
-    rake (0.8.7)
-    rbx-require-relative (0.0.5)
-    rcov (0.9.9)
+    net-scp (1.0.4)
+      net-ssh (>= 1.99.1)
+    net-ssh (2.1.4)
+    rake (0.9.2)
     rev (0.3.2)
       iobuffer (>= 0.1.3)
     rspec (2.6.0)
@@ -48,12 +67,22 @@ GEM
     rspec-expectations (2.6.0)
       diff-lcs (~> 1.1.2)
     rspec-mocks (2.6.0)
-    ruby-debug (0.10.4)
-      columnize (>= 0.1)
-      ruby-debug-base (~> 0.10.4.0)
-    ruby-debug-base (0.10.4)
-      linecache (>= 0.3)
+    ruby-debug-base19 (0.11.25)
+      columnize (>= 0.3.1)
+      linecache19 (>= 0.5.11)
+      ruby_core_source (>= 0.1.4)
+    ruby-debug19 (0.11.6)
+      columnize (>= 0.3.1)
+      linecache19 (>= 0.5.11)
+      ruby-debug-base19 (>= 0.11.19)
+    ruby_core_source (0.1.5)
+      archive-tar-minitar (>= 0.5.2)
+    simplecov (0.4.2)
+      simplecov-html (~> 0.4.4)
+    simplecov-html (0.4.5)
     thor (0.14.6)
+    virtualbox (0.9.1)
+      ffi (~> 1.0.9)
     yajl-ruby (0.8.2)
 
 PLATFORMS
@@ -63,7 +92,8 @@ DEPENDENCIES
   growl
   guard-rspec
   melissadata!
-  rake (= 0.8.7)
-  rcov
+  rake
   rspec (> 2.4)
-  ruby-debug
+  ruby-debug19
+  simplecov (>= 0.4.0)
+  vagrant!

data/README.md

@@ -1,5 +1,53 @@
 # MelissaData
 
-Ruby wrappers around MelissaData. Initial spike - pre alph ...
+Links to products/projects used:
 
-foo
+* MelissaData Data Quality Website: [http://www.melissadata.com/dqt/index.htm](http://www.melissadata.com/dqt/index.htm)
+* Vagrant: [http://vagrantup.com/](http://vagrantup.com/)
+* Oracle's VirtualBox: [VirtualBox](http://www.virtualbox.org)
+
+Ruby wrappers around the MelissaData objects. The code to access the raw
+shared objects is based on the sample code provided by the MelissaData
+DVDs / downloads.
+
+## Prereqs
+
+Make sure these are all installed on your development machine:
+
+* Oracle's VirtualBox: [VirtualBox](http://www.virtualbox.org)
+* RVM: [http://beginrescueend.com](http://beginrescueend.com)
+* Ruby 1.9.2 via RVM: `rvm install 1.9.2`
+* A MelissaData DVD or .dmg (e.g. IPL-DVD-2011-Q1 or DQ-DVD-2011-05)
+
+## Quick Start
+
+Once all the prereqs are installed, this should get you a running
+Vagrant VM:
+
+    git clone git://github.com/johnnyt/melissadata.git
+    cd melissadata
+    rake setup
+
+Now make sure you have a MelissaData DVD (or .dmg) mounted (e.g. IPL-DVD-2011-Q1), and run:
+
+    rake install_md
+
+Then SSH into the Vagrant VM:
+
+    bundle exec vagrant ssh
+
+Once logged into the VM - run:
+
+    cd /opt/melissadata/gem
+    irb -r lib/md
+
+Inside the IRB session run:
+
+    irb> ip = MelissaData::NativeObject::IpLocator.new :license => 'YOUR_LICENSE_KEY'
+    irb> ip.process :ip => '74.125.224.81'
+
+which should return:
+
+    => {:domain_name=>"google.com", :country_abbrev=>"US", :region=>"California", :zip=>"94043",
+        :isp=>"Google Inc", :lat=>"37.386052", :country=>"United States", :city=>"Mountain View",
+        :lng=>"-122.083851", :messages=>{:status=>["IP Address found in database"]}, :result_codes=>"IS01"}

data/Rakefile

@@ -5,42 +5,77 @@ task :setup do
   sh %Q!gem install bundler --no-ri --no-rdoc! unless bundler_installed
   output = `bundle check 2>&1`
   unless $?.to_i == 0
-    # puts output
     sh "bundle install"
-    # puts
   end
 
-  # Dir['config/*.example'].each do |example_file|
-  #   config_file = example_file.gsub(/\.example/,'')
-  #   sh %Q!cp #{example_file} #{config_file}! unless File.exists?(config_file)
-  # end
+  unless %x[bundle exec vagrant box list] =~ /lucid64$/
+    sh "bundle exec vagrant box add lucid64 http://files.vagrantup.com/lucid64.box"
+  end
+
+  sh "bundle exec vagrant up"
+
+  puts <<-STR
+
+
+    Now make sure you have a MelissaData DVD (or .dmg) mounted (e.g. IPL-DVD-2011-Q1),
+    then run:
 
-  puts "Done!\n\n"
+      rake md:install
+
+  STR
 end
 
 if bundler_installed
+  desc "SSH into the Vagrant VM"
+  task :ssh do
+    sh "bundle exec vagrant ssh"
+  end
+
+  task :console => 'md:console'
+
+  namespace :md do
+    desc "Open a MelissaData console (IRB session)"
+    task :console do
+      sh "bundle exec irb -r./lib/md"
+    end
+
+    desc "Install MelissaData onto the Vagrant VM"
+    task :install do
+      sh "bundle exec rake build"
+      sh "bundle exec vagrant md install"
+    end
+
+    desc "Update the MelissaData gem on the Vagrant VM"
+    task :update do
+      sh "bundle exec rake build"
+      sh "bundle exec vagrant md update"
+    end
+  end
+
   require 'bundler'
   Bundler::GemHelper.install_tasks
 
-  require 'rspec/core/rake_task'
+  if defined?(RSpec)
+    require 'rspec/core/rake_task'
 
-  task :default => [:spec]
-  task :test => [:spec]
+    task :default => [:spec]
+    task :test => [:spec]
 
-  desc "Run specs"
-  RSpec::Core::RakeTask.new('spec') do |t|
-    t.rspec_opts = ['--backtrace', '--format', 'documentation']
-    t.pattern = 'spec/**/*_spec.rb'
-  end
+    desc "Run specs"
+    RSpec::Core::RakeTask.new('spec') do |t|
+      t.rspec_opts = ['--backtrace', '--format', 'documentation']
+      t.pattern = 'spec/**/*_spec.rb'
+    end
 
-  if RUBY_VERSION !~ /^1\.9/
-    namespace :spec do
-      desc "Run specs with RCov"
-      RSpec::Core::RakeTask.new('rcov') do |t|
-        t.pattern = 'spec/**/*_spec.rb'
-        t.rspec_opts = ['--backtrace', '--format', 'documentation']
-        t.rcov = true
-        # t.rcov_opts = ['--exclude', 'some/path']
+    if RUBY_VERSION !~ /^1\.9/
+      namespace :spec do
+        desc "Run specs with RCov"
+        RSpec::Core::RakeTask.new('rcov') do |t|
+          t.pattern = 'spec/**/*_spec.rb'
+          t.rspec_opts = ['--backtrace', '--format', 'documentation']
+          t.rcov = true
+          # t.rcov_opts = ['--exclude', 'some/path']
+        end
       end
     end
   end

data/Vagrantfile

@@ -1,17 +1,17 @@
-Vagrant::Config.run do |config|
-  config.vm.box = 'melissa_data'
-  config.vm.forward_port 'http', 80, 8888
-  config.vm.forward_port 'melissa_data', 23456, 23457
+require File.expand_path('lib/melissadata', File.dirname(__FILE__))
+MelissaData::VagrantPlugin.load!
 
-  config.vm.customize do |vm|
-    vm.memory_size = 1024
-    vm.cpu_count = 2
+Vagrant::Config.run do |config|
+  config.vm.box = 'lucid64'
+  config.vm.provision :chef_solo do |chef|
+    chef.cookbooks_path = ['chef/cookbooks']
+    chef.add_recipe 'melissadata'
   end
 
-  # config.vm.provision :chef_solo do |chef|
-  #   chef.roles_path = '../aha_roles'
-  #   chef.cookbooks_path = ['../aha_cookbooks']
-  #   chef.add_role 'vagrant'
-  #   chef.add_recipe 'melissa_data'
+  config.vm.forward_port 'melissadata', 1404, 14004
+
+  # config.vm.customize do |vm|
+  #   vm.cpu_count = 2
+  #   vm.memory_size = 1024
   # end
 end

data/bin/md_tcp_server

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+require File.expand_path('../lib/melissadata/melissadata', File.dirname(__FILE__))
+
+svr = MelissaData::Server::TCP.new
+svr.run

data/bin/md_unix_server

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+require File.expand_path('../lib/melissadata/melissadata', File.dirname(__FILE__))
+
+svr = MelissaData::Server::Unix.new
+svr.run

data/chef/cookbooks/apt/README.md

@@ -0,0 +1,109 @@
+Description
+===========
+
+Configures various APT components on Debian-like systems.  Also includes a LWRP.
+
+Recipes
+=======
+
+default
+-------
+The default recipe runs apt-get update during the Compile Phase of the Chef run to ensure that the system's package cache is updated with the latest. It is recommended that this recipe appear first in a node's run list (directly or through a role) to ensure that when installing packages, Chef will be able to download the latest version available on the remote APT repository.
+
+This recipe also sets up a local cache directory for preseeding packages.
+
+cacher
+------
+Installs the apt-cacher package and service so the system can provide APT caching. You can check the usage report at http://{hostname}:3142/report. The cacher recipe includes the `cacher-client` recipe, so it helps seed itself.
+
+cacher-client
+-------------
+Configures the node to use the apt-cacher server as a client.
+
+Resources/Providers
+===================
+
+This LWRP provides an easy way to manage additional APT repositories.
+
+# Actions
+
+- :add: creates a repository file and builds the repository listing
+- :remove: removes the repository file
+
+# Attribute Parameters
+
+- repo_name: name attribute. The name of the channel to discover
+- uri: the base of the Debian distribution
+- distribution: this is usually your release's codename...ie something like `karmic`, `lucid` or `maverick`
+- components: package groupings..when it doubt use `main`
+- deb_src: whether or not to add the repository as a source repo as well
+- key_server: the GPG keyserver where the key for the repo should be retrieved
+- key: if a `key_server` is provided, this is assumed to be the fingerprint, otherwise it is the URI to the GPG key for the repo
+
+# Example
+
+    # add the Zenoss repo
+    apt_repository "zenoss" do
+      uri "http://dev.zenoss.org/deb"
+      components ["main","stable"]
+      action :add
+    end
+    
+    # add the Nginx PPA; grab key from keyserver
+    apt_repository "nginx-php" do
+      uri "http://ppa.launchpad.net/nginx/php5/ubuntu"
+      distribution node['lsb']['codename']
+      components ["main"]
+      keyserver "keyserver.ubuntu.com"
+      key "C300EE8C"
+      action :add
+    end
+    
+    # add the Cloudkick Repo
+    apt_repository "cloudkick" do
+      uri "http://packages.cloudkick.com/ubuntu"
+      distribution node['lsb']['codename']
+      components ["main"]
+      key "http://packages.cloudkick.com/cloudkick.packages.key"
+      action :add
+    end
+    
+    # remove Zenoss repo
+    apt_repository "zenoss" do
+      action :remove
+    end
+    
+Usage
+=====
+
+Put `recipe[apt]` first in the run list. If you have other recipes that you want to use to configure how apt behaves, like new sources, notify the execute resource to run, e.g.:
+
+    template "/etc/apt/sources.list.d/my_apt_sources.list" do
+      notifies :run, resources(:execute => "apt-get update"), :immediately
+    end
+
+The above will run during execution phase since it is a normal template resource, and should appear before other package resources that need the sources in the template.
+
+Put `recipe[apt::cacher]` in the run_list for a server to provide APT caching and add `recipe[apt::cacher-client]` on the rest of the Debian-based nodes to take advantage of the caching server.
+
+License and Author
+==================
+
+Author:: Joshua Timberman (<joshua@opscode.com>)
+Author:: Matt Ray (<matt@opscode.com>)
+Author:: Seth Chisamore (<schisamo@opscode.com>)
+
+Copyright 2009-2011 Opscode, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+

data/chef/cookbooks/apt/files/default/apt-cacher

@@ -0,0 +1,9 @@
+# apt-cacher startup configuration file
+
+# IMPORTANT: check the apt-cacher.conf file before using apt-cacher as daemon.
+
+# set to 1 to start the daemon at boot time
+AUTOSTART=1
+
+# extra settings to override the ones in apt-cacher.conf
+# EXTRAOPT=" daemon_port=3142 limit=30 "

data/chef/cookbooks/apt/files/default/apt-cacher.conf

@@ -0,0 +1,144 @@
+# This file has been modified by ./apt-proxy-to-apt-cacher
+# Some lines may have been appended at the bottom of this file
+# This file has been modified by /usr/share/apt-cacher/apt-proxy-to-apt-cacher
+# Some lines may have been appended at the bottom of this file
+#################################################################
+# This is the config file for apt-cacher. On most Debian systems
+# you can safely leave the defaults alone.
+#################################################################
+
+# cache_dir is used to set the location of the local cache. This can
+# become quite large, so make sure it is somewhere with plenty of space.
+cache_dir=/var/cache/apt-cacher
+
+# The email address of the administrator is displayed in the info page
+# and traffic reports.
+admin_email=root@localhost
+
+# For the daemon startup settings please edit the file /etc/default/apt-cacher.
+
+# Daemon port setting, only useful in stand-alone mode. You need to run the
+# daemon as root to use privileged ports (<1024).
+daemon_port = 3142
+
+# optional settings, user and group to run the daemon as. Make sure they have
+# sufficient permissions on the cache and log directories. Comment the settings
+# to run apt-cacher as the native user.
+group=www-data
+user=www-data
+
+# optional setting, binds the listening daemon to one specified IP. Use IP
+# ranges for more advanced configuration, see below.
+# daemon_addr=localhost
+
+# If your apt-cacher machine is directly exposed to the Internet and you are
+# worried about unauthorised machines fetching packages through it, you can
+# specify a list of IPv4 addresses which are allowed to use it and another
+# list of IPv4 addresses which aren't.
+# Localhost (127.0.0.1) is always allowed. Other addresses must be matched
+# by allowed_hosts and not by denied_hosts to be permitted to use the cache.
+# Setting allowed_hosts to "*" means "allow all".
+# Otherwise the format is a comma-separated list containing addresses,
+# optionally with masks (like 10.0.0.0/22), or ranges of addresses (two
+# addresses separated by a hyphen, no masks, like '192.168.0.3-192.168.0.56').
+allowed_hosts=*
+denied_hosts=
+
+# And similiarly for IPv6 with allowed_hosts_6 and denied_hosts_6.
+# Note that IPv4-mapped IPv6 addresses (::ffff:w.x.y.z) are truncated to
+# w.x.y.z and are handled as IPv4.
+allowed_hosts_6=fec0::/16
+denied_hosts_6=
+
+# This thing can be done by Apache but is much simplier here - limit access to
+# Debian mirrors based on server names in the URLs
+#allowed_locations=ftp.uni-kl.de,ftp.nerim.net,debian.tu-bs.de
+
+# Apt-cacher can generate usage reports every 24 hours if you set this
+# directive to 1. You can view the reports in a web browser by pointing
+# to your cache machine with '/apt-cacher/report' on the end, like this:
+#      http://yourcache.example.com/apt-cacher/report
+# Generating reports is very fast even with many thousands of logfile
+# lines, so you can safely turn this on without creating much 
+# additional system load.
+generate_reports=1
+
+# Apt-cacher can clean up its cache directory every 24 hours if you set
+# this directive to 1. Cleaning the cache can take some time to run
+# (generally in the order of a few minutes) and removes all package
+# files that are not mentioned in any existing 'Packages' lists. This
+# has the effect of deleting packages that have been superseded by an
+# updated 'Packages' list.
+clean_cache=1
+
+# The directory to use for apt-cacher access and error logs.
+# The access log records every request in the format:
+# date-time|client ip address|HIT/MISS/EXPIRED|object size|object name
+# The error log is slightly more free-form, and is also used for debug
+# messages if debug mode is turned on.
+# Note that the old 'logfile' and 'errorfile' directives are
+# deprecated: if you set them explicitly they will be honoured, but it's
+# better to just get rid of them from old config files.
+logdir=/var/log/apt-cacher
+
+# apt-cacher can use different methods to decide whether package lists need to
+# be updated,
+# A) looking at the age of the cached files
+# B) getting HTTP header from server and comparing that with cached data. This
+# method is more reliable and avoids desynchronisation of data and index files
+# but needs to transfer few bytes from the server every time somebody requests
+# the files ("apt-get update")
+# Set the following value to the maximum age (in hours) for method A or to 0
+# for method B
+expire_hours=0
+
+# Apt-cacher can pass all its requests to an external http proxy like
+# Squid, which could be very useful if you are using an ISP that blocks
+# port 80 and requires all web traffic to go through its proxy. The
+# format is 'hostname:port', eg: 'proxy.example.com:8080'.
+http_proxy=proxy.example.com:8080
+
+# Use of an external proxy can be turned on or off with this flag.
+# Value should be either 0 (off) or 1 (on).
+use_proxy=0
+
+# External http proxy sometimes need authentication to get full access. The
+# format is 'username:password'.
+http_proxy_auth=proxyuser:proxypass
+
+# Use of external proxy authentication can be turned on or off with this flag.
+# Value should be either 0 (off) or 1 (on).
+use_proxy_auth=0
+
+# Rate limiting sets the maximum bandwidth in bytes per second to use
+# for fetching packages. Syntax is fully defined in 'man wget'.
+# Use 'k' or 'm' to use kilobits or megabits / second: eg, 'limit=25k'.
+# Use 0 or a negative value for no rate limiting.
+limit=0
+
+# Debug mode makes apt-cacher spew a lot of extra debug junk to the
+# error log (whose location is defined with the 'logdir' directive).
+# Leave this off unless you need it, or your error log will get very
+# big. Acceptable values are 0 or 1.
+debug=0
+
+# Adapt the line in the usage info web page to match your server configuration
+# example_sources_line=deb&nbsp;http://<b>my.cacher.server:3142/</b>ftp.au.debian.org/debian&nbsp;unstable&nbsp;main&nbsp;contrib&nbsp;non-free
+
+# Print a 410 (Gone) HTTP message with the specified text when accessed via
+# CGI. Useful to tell users to adapt their sources.list files when the
+# apt-cacher server is beeing relocated (via apt-get's error messages while
+# running "update")
+#cgi_advise_to_use = Please use http://cacheserver:3142/ as apt-cacher access URL
+#cgi_advise_to_use = Server relocated. To change sources.list, run perl -pe "s,/apt-cacher\??,:3142," -i /etc/apt/sources.list
+
+# Server mapping - this allows to hide real server names behind virtual paths
+# that appear in the access URL. This method is known from apt-proxy. This is
+# also the only method to use FTP access to the target hosts. The syntax is simple, the part of the beginning to replace, followed by a list of mirror urls, all space separated. Multiple profile are separated by semicolons
+# path_map = debian ftp.uni-kl.de/pub/linux/debian ftp2.de.debian.org/debian ; ubuntu archive.ubuntu.com/ubuntu ; security security.debian.org/debian-security ftp2.de.debian.org/debian-security
+# Note that you need to specify all target servers in the allowed_locations
+# options if you make use of it. Also note that the paths should not overlap
+# each other. FTP access method not supported yet, maybe in the future.
+
+# extra setting from apt-proxy configuration
+path_map =  ubuntu us.archive.ubuntu.com/ubuntu ; ubuntu-security security.ubuntu.com/ubuntu ; debian debian.osuosl.org/debian/ ; security security.debian.org/debian-security