remi-domain-finder 0.1.0

data/NOTES

@@ -0,0 +1,41 @@
+TODO
+
+  put all classes within DomainFinder namespace to not class with other libraries
+
+  turn into gem (github-compatible)
+
+  make a domain-finder console with some  nice and easy built in commands for checking domains/etc
+
+  make it check and eval a .domainfinderrc or _domainfinderrc for easily overriding anything 
+  like the default domain-finder command and whatever else
+
+  make domain-auction (shortcut to domain-finder auction)
+
+  make a shoes-based GUI specific to the auction functionality ... ./bin/domain-auction (maybe?)
+
+NOTES
+
+Auction Date google query examples:
+
+"Auction Date" 01-15-2008 intitle:club site:whois.domaintools.com
+intitle optional
+
+"date should be atleast 2 weeks ago" (so it'll be in the cache, probably?)
+
+expected usages ...
+
+domain-finder auction
+domain-finder auction --date 01-01-2008
+domain-finder auction --date 01-01-2008..01-05-2008
+domain-finder auction --date 01-01-2008 --keyword blog
+domain-finder auction --date 01-01-2008 --keyword blog,seo
+domain-finder auction --date 01-01-2008,01-04-2008..01-05-2008 --keyword blog,seo
+
+Google usage ...
+
+Google.search %{"Auction Date" blah ...}, :page => 2 should return an array of GoogleResult objects
+
+Google.search_url(....).should == "http://q=....."
+Google.fetch_url(...).should == string
+Google.parse_results(str).should == []
+Google.parse_result(hpricot thing).should == GoogleResult

data/README

@@ -0,0 +1,19 @@
+domain-finder
+=============
+
+domain-finder is a little utility for checking the availability 
+of domains and, specifically, for showing some recently used but 
+no longer registered domain names, using a technique developed 
+by Mark Fulton of dotsauce.com
+
+
+See NOTES for current TODO and future ideal example usage
+
+
+License
+-------
+
+domain-finder is released under the terms of the GNU Affero General Public License
+
+We reserve the right to re-release under a more permissive license in the future, 
+which will likely be the LGPL, allowing for library use by proprietery applications

data/Rakefile

@@ -0,0 +1,9 @@
+desc "Generate and view specdoc"
+task :specdoc do
+  system "spec -f specdoc spec/*_spec.rb" # kills BASH colors  :(
+end
+
+desc "Generate and view HTML specdoc"
+task :spechtml do
+  system "spec -f html spec/*_spec.rb > tmp/specdoc.html; firefox tmp/specdoc.html"
+end

data/bin/domain-finder

@@ -0,0 +1,4 @@
+#! /usr/bin/env ruby
+require File.dirname(__FILE__) + '/../lib/domain-finder'
+require 'domain-finder/bin'
+DomainFinder::Bin.new( ARGV ).run

data/domain-finder.gemspec

@@ -0,0 +1,22 @@
+Gem::Specification.new do |s|
+  s.name = "domain-finder"
+  s.version = "0.1.0"
+  s.date = "2008-06-14"
+  s.summary = "Find available domains names"
+  s.email = "remi@remitaylor.com"
+  s.homepage = "http://github.com/remi/domain-finder"
+  s.description = "Find some previously used domain names and check availability"
+  s.has_rdoc = true
+  s.rdoc_options = ["--quiet", "--title", "domain-finder", "--opname", "index.html", "--line-numbers", "--main", "README", "--inline-source"]
+  s.extra_rdoc_files = ['README']
+  s.authors = ["remi Taylor"]
+
+  # generate using: $ ruby -e "puts Dir['**/**'].select{|x| File.file?x}.inspect"
+  s.files = ["bin/domain-finder", "COPYING", "lib/domain-finder.rb", "lib/domain-finder/bin.rb", "lib/domain-finder/domaintools.rb", "lib/domain-finder/google.rb", "lib/domain-finder/daterange.rb", "lib/domain-finder/moniker.rb", "lib/domain-finder/domain.rb", "lib/domain-finder/secure_post.rb", "domain-finder.gemspec", "Rakefile", "NOTES", "README", "tmp/specdoc.html", "spec/spec_helper.rb", "spec/google_spec.rb", "spec/moniker_spec.rb", "spec/daterange_spec.rb", "spec/html/google-1.html", "spec/html/moniker-1.html", "spec/html/domaintools-1.html", "spec/domaintools_spec.rb"]
+
+  s.add_dependency("remi-simplecli", ["> 0.0.0"])
+  s.add_dependency("hpricot", ["> 0.0.0"])
+  
+  s.executables = ["domain-finder"]
+  s.default_executable = "domain-finder"
+end

data/lib/domain-finder.rb

@@ -0,0 +1,6 @@
+$:.unshift File.dirname(__FILE__)
+
+class DomainFinder; end
+
+%w( rubygems open-uri net/https hpricot date ).each { |lib| require lib }
+%w( secure_post google domaintools moniker domain daterange ).each { |lib| require "domain-finder/#{lib}" }

data/lib/domain-finder/bin.rb

@@ -0,0 +1,113 @@
+require 'optparse'
+require 'simplecli'
+
+class DomainFinder::Bin
+  include SimpleCLI
+
+  def usage *args
+    puts <<doco
+
+  domain-finder == %{ domain availability utility }
+
+    Usage:
+      domain-finder -h/--help      [Not Implemented Yet]
+      domain-finder -v/--version   [Not Implemented Yet]
+      domain-finder command [options]
+
+    Examples:
+      domain-finder auction
+
+    Further help:
+      domain-finder commands         # list all available commands
+      domain-finder help <COMMAND>   # show help for COMMAND
+      domain-finder help             # show this help message
+
+doco
+  end 
+
+  def available_help
+    <<doco
+Usage: #{ script_name } available domain.com[ domain2.com]
+
+    Summary:
+      Check the availability of domains (max 500)
+doco
+  end
+  def available *domains
+    available_domains = Domain.available? *domains
+
+    unless available_domains.empty?
+      puts "\nAvailable Domains:\n------------------"
+      available_domains.sort.each do |domain|
+        puts "- #{domain}"
+      end
+    else
+      puts "No Domains Available"
+    end
+  end
+
+  def auction_help
+    <<doco
+Usage: #{ script_name } auction
+
+    Options:
+      -d, --dates         Set of dates to use in search
+      -k, --keywords      Comma-delimited keywords for domain titles
+      -p, --pages         Number of Google pages to gather domains from
+      -f, --file          Name of a text file to save available domain names to
+
+    Examples:
+      #{ script_name } auction --dates 05-01-2008,06-10-2008..06.15.2008
+      #{ script_name } auction --keywords seo,blog --dates 06-01-2008..07-01-2008
+      #{ script_name } auction -f domains.txt --keywords cat,dog -p 2
+
+    Summary:
+      Display available auction domains
+doco
+  end
+  def auction *args
+    begin
+      options = { :dates => [Date.today - 14], :pages => 1 }
+      opts = OptionParser.new do |opts|
+        opts.on('-d','--dates DATE'){ |d| options[:dates] = DateRange.parse(d) }
+        opts.on('-k','--keywords WORDS'){ |w| options[:keywords] = w.split(',') }
+        opts.on('-p','--pages PAGES'){ |p| options[:pages] = p.to_i }
+        opts.on('-f','--file FILE'){ |f| options[:file] = f }
+      end
+      opts.parse! args
+
+      query =  %{"Auction Date" #{ options[:dates].join(' OR ') } site:whois.domaintools.com }
+      query += options[:keywords].map { |k| "intitle:#{k}" }.join(' OR ') if options[:keywords]
+
+      puts "Searching google ... [#{query}]"
+      results = Google.search(query, :pages => options[:pages] )
+
+      print "Getting domains ."
+      auction_domains = results.inject([]) do |all,result|
+        if result.cached
+          print '.'
+          all + DomainTools.auction_domains( open(result.cached) )
+        else
+          all # no google cache for this result, continue on to the next result
+        end
+      end
+      print "\n"
+      
+      puts "Checking availability of #{ auction_domains.length } domains ..."
+      available_domains = Domain.available? *auction_domains
+      
+      unless available_domains.empty?
+        puts "\nAvailable Domains:\n------------------"
+        available_domains.sort.each do |domain|
+          puts "- #{domain}"
+        end
+        File.open(options[:file],'w'){|f| f << available_domains.sort.join("\n") } if options[:file]
+      else
+        puts "No Domains Available"
+      end
+    rescue => ex
+      puts "Exception: #{ex.inspect}"
+    end
+  end
+
+end

data/lib/domain-finder/daterange.rb

@@ -0,0 +1,25 @@
+require 'date'
+
+# Change default Date#to_s format
+class Date
+  def to_s
+    self.strftime '%m-%d-%Y'
+  end
+end
+
+# Simple class for helping to parse string sets of dates and date ranges
+class DateRange
+
+  def self.parse text
+    dates = text.gsub('-','/').split(',')
+    dates.inject([]) do |all,this|
+      if this.include?'..'
+        range_dates = this.split('..')
+        all + (Date.parse(range_dates.first)..Date.parse(range_dates.last)).to_a
+      else
+        all << Date.parse(this)
+      end
+    end
+  end
+
+end

data/lib/domain-finder/domain.rb

@@ -0,0 +1,9 @@
+# Simple class for checking the availability of domains
+# (abstracts specific website in use so API doesn't change if we switch websites)
+class Domain
+
+  def self.available? *domains
+    Moniker.find_available *domains
+  end
+
+end

data/lib/domain-finder/domaintools.rb

@@ -0,0 +1,8 @@
+# Simple class for parsing a DomainTools page
+class DomainTools
+
+  def self.auction_domains html
+    ( Hpricot(html) / 'div.ajax.12 tr td a' ).map { |a| a.inner_text.strip }.select { |txt| not txt.empty? }
+  end
+
+end

data/lib/domain-finder/google.rb

@@ -0,0 +1,53 @@
+# Simple class for searching google and getting results
+class Google
+
+  BASE_SEARCH_URL = "http://www.google.com/search?q="
+
+  def self.search_url query, options = {}
+    url =  BASE_SEARCH_URL + URI.encode(query)
+    url += "&start=#{ (options[:page] - 1) * 10 }" if options[:page] and options[:page] != 1
+    url
+  end
+
+  def self.fetch_url url
+    open(url).read
+  end
+
+  def self.parse_results html
+    ( Hpricot(html) / 'div.g' ).map { |result| Result.new(result) }
+  end
+
+  def self.search query, options = {}
+    if options[:pages] and options[:pages] > 1
+      results = []
+      options[:pages].times do |i|
+        results += parse_results( fetch_url(search_url(query, :page => i+1)) )
+      end
+      results
+    else
+      parse_results( fetch_url(search_url(query)) )
+    end
+  end
+
+  # Simple class representing a single Google result
+  class Result
+    attr_accessor :title, :url, :snippet, :cached
+
+    def initialize result
+      @title   = ( result / 'h2.r' ).inner_text
+      @url     = ( result / 'h2.r a' )[0]['href']
+
+      cached_element = ( result / 'div.std nobr a' )[0]
+      @cached  = cached_element['href'] if cached_element and cached_element.inner_text == 'Cached'
+
+      snippet_element = ( result / 'div.std' )[0]
+      if snippet_element
+        @snippet = ""
+        snippet_element.children.each do |this| 
+          break if this.to_s == '<br />'
+          @snippet << this.inner_text
+        end 
+      end
+    end
+  end
+end

data/lib/domain-finder/moniker.rb

@@ -0,0 +1,17 @@
+# Simple class for batch searching domain availability
+class Moniker
+
+  INTERNATIONAL_TEXT = "Select language  Chinese Afrikaans Albanian Arabic Aragonese Armenian Assamese Asturian Avestan Awadhi Azerbaijani Balinese Baluchi Basa Bashkir Basque Belarusian Bengali Bhojpuri Bosnian Bulgarian Burmese Carib Catalan Chechen Chinese Chuvash Coptic Corsican Croatian Czech Danish Divehi Dogri Dutch English Estonian Fijian Finnish French Frisian Gaelic Georgian German Gondi Greek Gujarati Hebrew Hindi Hungarian Icelandic Indic Indonesian Ingush Irish Italian Japanese Javanese Kashmiri Kazakh Khmer Kirghiz Korean Kurdish Lao Latvian Lithuanian Luxembourgish Macedonian Malay Malayalam Maltese Maori Moldavian Nepali Norwegian Oriya Ossetian Panjabi Persian Polish Portuguese Pushto Rajasthani Romanian Russian Samoan Sanskrit Sardinian Serbian Sindhi Sinhalese Slovak Slovenian Somali Spanish Swahili Swedish Syriac Tajik Tamil Telugu Thai Tibetan Turkish Ukrainian Urdu Uzbek Vietnamese Welsh Yiddish"
+
+  def self.find_available *domains
+    data = { :cmd => 'check', :scope => 'batch', :domnamelist => domains.join("\n") }
+    parse_results SecurePost.post('https://www.moniker.com/pub/DomainCheckBatch', data)
+  end
+
+  def self.parse_results html
+    ( Hpricot(html) / "td[@style='font-family: verdana; font-size: 8pt;']").map do 
+      |td| td.inner_text.gsub('?','').sub(INTERNATIONAL_TEXT,'').strip
+    end
+  end
+
+end

data/lib/domain-finder/secure_post.rb

@@ -0,0 +1,12 @@
+# Simple class for helping to easily make a secure POST without messing with Net::HTTP
+class SecurePost
+  
+  def self.post url, data = {}
+    uri = URI.parse url
+    http = Net::HTTP.new uri.host, 443
+    http.use_ssl = true
+    resp, data   = http.post(uri.path, data.map{ |k,v| "#{k}=#{v}" }.join('&'), {})
+    data
+  end
+
+end

data/spec/daterange_spec.rb

@@ -0,0 +1,35 @@
+require File.dirname(__FILE__) + '/spec_helper.rb'
+
+describe DateRange do
+
+  it '.parse should parse a single date' do
+    DateRange.parse("01-31-2008").first.should == Date.parse("01/31/2008")
+    DateRange.parse("01/31/2008").first.should == Date.parse("01/31/2008")
+  end
+
+  it '.parse should parse comma delimited dates' do
+    dates = DateRange.parse("01-15-2008,01-31-2008")
+    dates.length.should == 2
+    dates.first.should == Date.parse("01/15/2008")
+    dates.last.should  == Date.parse("01/31/2008")
+  end
+
+  it '.parse should parse a date range' do
+    dates = DateRange.parse("01-10-2008..01-14-2008")
+    dates.length.should == 5
+    dates.first.should == Date.parse("01/10/2008")
+    dates[1].should    == Date.parse("01/11/2008")
+    dates.last.should  == Date.parse("01/14/2008")
+  end
+
+  it '.parse should parse comma delimited dates and date ranges' do
+    dates = DateRange.parse("01-10-2008..01-14-2008,02-01-2008,06-15-2005,12-20-1000..12-24-1000")
+    dates.length.should == 12
+    dates.first.should  == Date.parse("01/10/2008")
+    dates.last.should   == Date.parse("12/24/1000")
+    %w( 01/13/2008 02/01/2008 06/15/2005 12/21/1000 ).each do |date|
+      dates.should include(Date.parse(date))
+    end
+  end
+
+end

data/spec/domaintools_spec.rb

@@ -0,0 +1,15 @@
+require File.dirname(__FILE__) + '/spec_helper.rb'
+
+describe DomainTools do
+
+  before(:all) do
+    @example_html  = File.read(File.dirname(__FILE__) + "/html/domaintools-1.html")
+  end
+
+  it '.auction_domains should return Domains at Auction' do
+    domains = DomainTools.auction_domains(@example_html)
+    domains.length.should == 13
+    domains.should include("EcoRus.net")
+  end
+
+end

data/spec/google_spec.rb

@@ -0,0 +1,54 @@
+require File.dirname(__FILE__) + '/spec_helper.rb'
+
+describe Google do
+
+  before(:all)do
+    @example_query = %{site:whois.domaintools.com "Auction Date" 06-01-2008}
+    @example_url   = %{http://www.google.com/search?q=site:whois.domaintools.com%20%22Auction%20Date%22%2006-01-2008}
+    @example_html  = File.read(File.dirname(__FILE__) + "/html/google-1.html")
+  end
+
+  it '.search_url should generate an escaped search url' do
+    Google.search_url(@example_query).should == @example_url
+  end
+
+  it '.search_url should support pagination' do
+    Google.search_url(@example_query, :page => 1 ).should == @example_url
+    Google.search_url(@example_query, :page => 2 ).should == @example_url + '&start=10'
+    Google.search_url(@example_query, :page => 10).should == @example_url + '&start=90'
+  end
+
+  it '.fetch_url should fetch the results of a full url' do
+    Google.should_receive(:open).with(@example_url).and_return(StringIO.new(@example_html))
+    Google.fetch_url(@example_url).should == @example_html
+  end
+
+  it '.parse_results should return array of GoogleResult objects' do
+    results = Google.parse_results @example_html
+    results.length.should == 10
+    first = results.first
+
+    first.title.should   == "Auctionrus.com - Auction Rus"
+    first.url.should     == "http://whois.domaintools.com/auctionrus.com"
+    first.cached.should  == %{http://209.85.173.104/search?q=cache:f7qbaIH-UTEJ:whois.domaintools.com/auctionrus.com+site:whois.domaintools.com+%22Auction+Date%22+06-01-2008&hl=en&ct=clnk&cd=1&gl=us&ie=UTF-8}
+    first.snippet.should == "Domain, Auction Date. EcoRus.net, 06-01-2008. PlusRus.com, 06-01-2008. Exclusiv-Rus.com, 06-01-2008. SermonsRus.com, 06-01-2008. HaUroRus.com, 06-01-2008 ..."
+  end
+
+  it '.search should return GoogleResult objects for a query' do
+    Google.should_receive(:open).with(@example_url).and_return(StringIO.new(@example_html))
+    results = Google.search(@example_query)
+    results.length.should == 10
+    results.first.should be_a_kind_of(Google::Result)
+  end
+  
+  it '.search should support pagination' do
+    Google.should_receive(:open).with(@example_url).and_return(StringIO.new(@example_html))
+    Google.should_receive(:open).with(@example_url + '&start=10').and_return(StringIO.new(@example_html))
+    Google.should_receive(:open).with(@example_url + '&start=20').and_return(StringIO.new(@example_html))
+
+    results = Google.search(@example_query, :pages => 3)
+    results.length.should == 30
+    results.first.should be_a_kind_of(Google::Result)
+  end
+
+end