iZsh-wwmd 0.2.19

data/lib/wwmd/wwmd_config.rb

@@ -0,0 +1,52 @@
+module WWMD
+
+  class WWMDConfig#:nodoc:
+    # for backward compat
+    def self.load_config(file); WWMD::load_config(file); end
+    def self.parse_opts(args); WWMD::parse_opts(args); end
+  end
+
+  def load_config(file)
+    begin
+      config = YAML.load_file(file)
+    rescue => e
+      putw "config file not found #{file}"
+      putw e.inspect
+      exit
+    end
+    return config
+  end
+
+  def parse_opts(args)
+    inopts = Hash.new
+    inopts[:max_redirects] = 10
+    inopts[:timeout] = 30
+    inopts[:scrape_warn] = false
+    opts = OptionParser.new do |opts|
+    # set defaults
+      opts.on("-p", "--password PASSWORD", "Password")     { |v| inopts[:password] = v }
+      opts.on("-u", "--username USERNAME", "Username")     { |v| inopts[:username] = v }
+      opts.on("--header_file HEADER_FILE","Header file")   { |v| inopts[:header_file] = v }
+      opts.on("--base_url BASE_URL","Base url")            { |v| inopts[:base_url] = v }
+      opts.on("--use_proxy PROXY_URL", "Use proxy at url") do |v|
+        ENV['HTTP_PROXY'] = "http://" + v.to_s
+        inopts[:use_proxy] = true
+        inopts[:proxy_url] = v
+      end
+      opts.on("--no_proxy","do not use proxy") do |v|
+        inopts[:use_proxy] = false
+        inopts[:proxy_url] = nil
+      end
+      opts.on("--use_auth","login before getting url")     { |v| inopts[:use_auth] = true }
+      opts.on("--no_auth","no login before getting url")   { |v| inopts[:use_auth] = false }
+      opts.on("--debug","debugging really doesn't work")   { |v| inopts[:debug] = true }
+      opts.on_tail("-h", "--help", "Show this message") do
+        puts opts
+        exit
+      end
+    end
+    opts.parse!(args)
+    return inopts
+  end
+
+end

data/lib/wwmd/wwmd_puts.rb

@@ -0,0 +1,9 @@
+module WWMD
+  attr_accessor :console
+  attr_accessor :debug
+  @console = false
+  @debug = false
+  def putd(*args); puts *args if WWMD::debug; end
+  def putx(*args); puts *args if WWMD::console; end
+  def putw(*args); puts *args if WWMD::console; end
+end

data/lib/wwmd/wwmd_utils.rb

@@ -0,0 +1,28 @@
+module WWMDUtils
+
+  def self.header_array_from_file(filename)
+    ret = Hash.new
+    File.readlines(filename).each do |line|
+      a = line.chomp.split(/\t/,2)
+      ret[a[0]] = a[1]
+    end
+    return ret
+  end
+
+  def self.ranstr(len=8,digits=false)
+    chars = ("a".."z").to_a
+    chars += ("0".."9").to_a if digits
+    ret = ""
+    1.upto(len) { |i| ret << chars[rand(chars.size-1)] }
+    return ret
+  end
+
+  def self.rannum(len=8,hex=false)
+    chars = ("0".."9").to_a
+    chars += ("A".."F").to_a if hex
+    ret = ""
+    1.upto(len) { |i| ret << chars[rand(chars.size-1)] }
+    return ret
+  end
+
+end

data/spec/README

@@ -0,0 +1,3 @@
+None of this was developed with unit tests
+
+These are tests written against new functionality

data/spec/form_array.spec

@@ -0,0 +1,49 @@
+#!/usr/bin/env ruby
+require 'wwmd'
+include WWMD
+require 'spec'
+
+describe FormArray do
+  before(:each) do
+    @form = FormArray.new
+  end
+
+  it "sets a value and reads a value" do
+    @form["foo"] = "bar"
+    @form["foo"].should == "bar"
+  end
+
+  it "reads from a string" do
+    @form = "foo=bar&baz=eep&argle=bargle".to_form
+    @form["foo"].should == "bar"
+    @form["baz"].should == "eep"
+    @form["argle"].should == "bargle"
+  end
+
+  it "to_get" do
+    str = "foo=bar&baz=eep&argle=bargle"
+    get = "?" + str
+    @form = str.to_form
+    @form.to_get.should == get
+  end
+
+  it "remove_nulls!" do
+    @form["var1"] = "not null"
+    @form["var2"] = ""
+    @form["var3"] = nil
+    @form.remove_nulls!
+    @form.size.should == 1
+    @form["var1"].should == "not null"
+  end
+
+  it "clones correctly" do
+    @form = "foo=bar&baz=eep&argle=bargle".to_form
+    lform = @form.clone
+    lform["foo"] = "test"
+    @form["foo"].should == "bar"
+    lform["foo"].should == "test"
+  end
+
+  it "escapes characters correctly"
+  it "unescapes characters correctly"
+end

data/spec/spider_csrf_test.spec

@@ -0,0 +1,28 @@
+#!/usr/bin/env ruby
+require 'wwmd'
+include WWMD
+require 'spec'
+
+describe Page do
+  before(:each) do
+    @page = Page.new({:base_url => "http://localhost"})
+    @spider = @page.spider
+    @spider.csrf_token = "CsRf"
+  end
+
+  it "should remove csrf tokens from visited and queued" do
+    url = "http://localhost/foo.php?CsRf=something&bar=baz"
+    links = ["http://localhost/q1.php?CsRf=omg&first=FIRST"]
+    @spider.add(url,links)
+    @spider.visited.first.should == "http://localhost/foo.php?CsRf=&bar=baz"
+    @spider.queued.first.should == "http://localhost/q1.php?CsRf=&first=FIRST"
+  end
+
+  it "should work normally" do
+    url = "http://localhost/foo.php?hithere=something&bar=baz"
+    links = ["http://localhost/q1.php?hithere=omg&first=FIRST"]
+    @spider.add(url,links)
+    @spider.visited.first.should == "http://localhost/foo.php?hithere=something&bar=baz"
+    @spider.queued.first.should == "http://localhost/q1.php?hithere=omg&first=FIRST"
+  end
+end

data/spec/urlparse_test.spec

@@ -0,0 +1,101 @@
+#!/usr/bin/env ruby
+require 'wwmd/urlparse'
+include WWMD
+require 'spec'
+
+describe URLParse do
+  before(:each) do
+    @base = "https://www.base.com"
+    @up = URLParse.new()
+  end
+
+  it "should parse the basic case" do
+    @up.parse("https://www.location.com/","/path/path/path/script.scr").to_s.should \
+      == "https://www.location.com/path/path/path/script.scr"
+    @up.proto.should == "https"
+    @up.location.should == "www.location.com"
+    @up.path.should == "/path/path/path/"
+    @up.script.should == "script.scr"
+    @up.parse("https://www.location.com/","/path/path/path/script").to_s.should \
+      == "https://www.location.com/path/path/path/script"
+    @up.proto.should == "https"
+    @up.location.should == "www.location.com"
+    @up.path.should == "/path/path/path/script"
+#    @up.script.should == nil
+    @up.script.should == ""
+  end
+
+  it "should parse when complete urls are passed" do
+    @up.parse(@base,"https://www.location.com/hithere/dirname/test.php").to_s.should \
+      == "https://www.location.com/hithere/dirname/test.php"
+    @up.proto.should == "https"
+    @up.location.should == "www.location.com"
+    @up.path.should == "/hithere/dirname/"
+    @up.script.should == "test.php"
+  end
+
+  it "should parse GET params correctly" do
+    @up.parse(@base,"http://www.location.com/test.php?foo=bar&baz=eep").to_s.should \
+      == "http://www.location.com/test.php?foo=bar&baz=eep"
+  end
+
+  it "should return the path if the path is fully qualified" do
+    @up.parse(@base,"http://www.location.com/").to_s.should == "http://www.location.com/"
+    @up.parse(@base,"http://www.location.com").to_s.should  == "http://www.location.com/"
+  end
+
+  it "should parse a location + relative link" do
+    @up.parse("https://www.location.com","relative/script.scr").to_s.should \
+      == "https://www.location.com/relative/script.scr"
+  end
+
+  it "should parse base urls with scripts (page.cur) + relative link" do
+    @up.parse("https://www.location.com/path/to/a_script.php", "more/script.scr").to_s.should == "https://www.location.com/path/to/more/script.scr"
+  end
+
+  it "should parse base urls without scripts + relative link" do
+    @up.parse("https://www.location.com/path/to/end", "but/more/script.scr").to_s.should \
+      == "https://www.location.com/path/to/end/but/more/script.scr"
+  end
+
+  it "should handle trailing slashes correctly" do
+    @up.parse(@base + "/","/test.php").to_s.should == "#{@base}/test.php"
+    @up.parse(@base + "/","/test.php").to_s.should_not == "#{@base}//test.php"
+  end
+
+  it "should parse dotdot correctly" do
+    @up.parse(@base + "/one/two/thee/four","../../foo.php").to_s.should \
+      == "#{@base}/one/two/foo.php"
+    @up.parse(("https://www.location.com/relative///path//deep/one"),"more/..//stuff/../foo.php").to_s.should \
+      == "https://www.location.com/relative/path/deep/one/foo.php"
+    @up.parse("https://www.location.com/rel/path/foo.php","../../../../../../bar.php").to_s.should \
+      == "https://www.location.com/bar.php"
+  end
+
+  it "should parse dot correctly" do
+    @up.parse("https://www.location.com","base/./../foo/././bar/script.scr").to_s.should \
+      == "https://www.location.com/foo/bar/script.scr"
+  end
+
+  it "should remove get params when posting to a form action with get params" do
+    @up.parse("https://www.location.com/mail/h/1nyas6k8hplt9/?s=t","?s=t&at=xn3j38mvpzxqd138zgwsooxvojvbvd").to_s.should \
+      == "https://www.location.com/mail/h/1nyas6k8hplt9/?s=t&at=xn3j38mvpzxqd138zgwsooxvojvbvd"
+  end
+
+  it "should not remove directory traversal params" do
+    @up.parse("http://www.example.com/?file=../../../../../../etc/passwd&param1=foobar.log&param2=false").to_s.should \
+      == "http://www.example.com/?file=../../../../../../etc/passwd&param1=foobar.log&param2=false"
+  end
+
+  it "should not remove directory traversal params 2" do
+    @up.parse("http://www.example.com:8888/foobar/barBaz.do?logFile=../../../../../../../../../../../../etc/passwd&foo=foobar.log&bazeep=false").to_s.should \
+      == "http://www.example.com:8888/foobar/barBaz.do?logFile=../../../../../../../../../../../../etc/passwd&foo=foobar.log&bazeep=false"
+  end
+
+  it "should not remove directory traversal params 2" do
+    @up.parse("http://www.example.com:8888", "/foobar/barBaz.do?logFile=../../../../../../../../../../../../etc/passwd&foo=foobar.log&bazeep=false").to_s.should \
+      == "http://www.example.com:8888/foobar/barBaz.do?logFile=../../../../../../../../../../../../etc/passwd&foo=foobar.log&bazeep=false"
+  end
+
+end
+

data/tasks/ann.rake

@@ -0,0 +1,80 @@
+
+begin
+  require 'bones/smtp_tls'
+rescue LoadError
+  require 'net/smtp'
+end
+require 'time'
+
+namespace :ann do
+
+  # A prerequisites task that all other tasks depend upon
+  task :prereqs
+
+  file PROJ.ann.file do
+    ann = PROJ.ann
+    puts "Generating #{ann.file}"
+    File.open(ann.file,'w') do |fd|
+      fd.puts("#{PROJ.name} version #{PROJ.version}")
+      fd.puts("    by #{Array(PROJ.authors).first}") if PROJ.authors
+      fd.puts("    #{PROJ.url}") if PROJ.url.valid?
+      fd.puts("    (the \"#{PROJ.release_name}\" release)") if PROJ.release_name
+      fd.puts
+      fd.puts("== DESCRIPTION")
+      fd.puts
+      fd.puts(PROJ.description)
+      fd.puts
+      fd.puts(PROJ.changes.sub(%r/^.*$/, '== CHANGES'))
+      fd.puts
+      ann.paragraphs.each do |p|
+        fd.puts "== #{p.upcase}"
+        fd.puts
+        fd.puts paragraphs_of(PROJ.readme_file, p).join("\n\n")
+        fd.puts
+      end
+      fd.puts ann.text if ann.text
+    end
+  end
+
+  desc "Create an announcement file"
+  task :announcement => ['ann:prereqs', PROJ.ann.file]
+
+  desc "Send an email announcement"
+  task :email => ['ann:prereqs', PROJ.ann.file] do
+    ann = PROJ.ann
+    from = ann.email[:from] || Array(PROJ.authors).first || PROJ.email
+    to   = Array(ann.email[:to])
+
+    ### build a mail header for RFC 822
+    rfc822msg =  "From: #{from}\n"
+    rfc822msg << "To: #{to.join(',')}\n"
+    rfc822msg << "Subject: [ANN] #{PROJ.name} #{PROJ.version}"
+    rfc822msg << " (#{PROJ.release_name})" if PROJ.release_name
+    rfc822msg << "\n"
+    rfc822msg << "Date: #{Time.new.rfc822}\n"
+    rfc822msg << "Message-Id: "
+    rfc822msg << "<#{"%.8f" % Time.now.to_f}@#{ann.email[:domain]}>\n\n"
+    rfc822msg << File.read(ann.file)
+
+    params = [:server, :port, :domain, :acct, :passwd, :authtype].map do |key|
+      ann.email[key]
+    end
+
+    params[3] = PROJ.email if params[3].nil?
+
+    if params[4].nil?
+      STDOUT.write "Please enter your e-mail password (#{params[3]}): "
+      params[4] = STDIN.gets.chomp
+    end
+
+    ### send email
+    Net::SMTP.start(*params) {|smtp| smtp.sendmail(rfc822msg, from, to)}
+  end
+end  # namespace :ann
+
+desc 'Alias to ann:announcement'
+task :ann => 'ann:announcement'
+
+CLOBBER << PROJ.ann.file
+
+# EOF

data/tasks/bones.rake

@@ -0,0 +1,20 @@
+
+if HAVE_BONES
+
+namespace :bones do
+
+  desc 'Show the PROJ open struct'
+  task :debug do |t|
+    atr = if t.application.top_level_tasks.length == 2
+      t.application.top_level_tasks.pop
+    end
+
+    if atr then Bones::Debug.show_attr(PROJ, atr)
+    else Bones::Debug.show PROJ end
+  end
+
+end  # namespace :bones
+
+end  # HAVE_BONES
+
+# EOF

data/tasks/gem.rake

@@ -0,0 +1,201 @@
+
+require 'find'
+require 'rake/packagetask'
+require 'rubygems/user_interaction'
+require 'rubygems/builder'
+
+module Bones
+class GemPackageTask < Rake::PackageTask
+  # Ruby GEM spec containing the metadata for this package.  The
+  # name, version and package_files are automatically determined
+  # from the GEM spec and don't need to be explicitly provided.
+  #
+  attr_accessor :gem_spec
+
+  # Tasks from the Bones gem directory
+  attr_reader :bones_files
+
+  # Create a GEM Package task library.  Automatically define the gem
+  # if a block is given.  If no block is supplied, then +define+
+  # needs to be called to define the task.
+  #
+  def initialize(gem_spec)
+    init(gem_spec)
+    yield self if block_given?
+    define if block_given?
+  end
+
+  # Initialization tasks without the "yield self" or define
+  # operations.
+  #
+  def init(gem)
+    super(gem.name, gem.version)
+    @gem_spec = gem
+    @package_files += gem_spec.files if gem_spec.files
+    @bones_files = []
+
+    local_setup = File.join(Dir.pwd, %w[tasks setup.rb])
+    if !test(?e, local_setup)
+      Dir.glob(::Bones.path(%w[lib bones tasks *])).each {|fn| bones_files << fn}
+    end
+  end
+
+  # Create the Rake tasks and actions specified by this
+  # GemPackageTask.  (+define+ is automatically called if a block is
+  # given to +new+).
+  #
+  def define
+    super
+    task :prereqs
+    task :package => ['gem:prereqs', "#{package_dir_path}/#{gem_file}"]
+    file "#{package_dir_path}/#{gem_file}" => [package_dir_path] + package_files + bones_files do
+      when_writing("Creating GEM") {
+        chdir(package_dir_path) do
+          Gem::Builder.new(gem_spec).build
+          verbose(true) {
+            mv gem_file, "../#{gem_file}"
+          }
+        end
+      }
+    end
+
+    file package_dir_path => bones_files do
+      mkdir_p package_dir rescue nil
+
+      gem_spec.files = (gem_spec.files +
+          bones_files.map {|fn| File.join('tasks', File.basename(fn))}).sort
+
+      bones_files.each do |fn|
+        base_fn = File.join('tasks', File.basename(fn))
+        f = File.join(package_dir_path, base_fn)
+        fdir = File.dirname(f)
+        mkdir_p(fdir) if !File.exist?(fdir)
+        if File.directory?(fn)
+          mkdir_p(f)
+        else
+          raise "file name conflict for '#{base_fn}' (conflicts with '#{fn}')" if test(?e, f)
+          safe_ln(fn, f)
+        end
+      end
+    end
+  end
+  
+  def gem_file
+    if @gem_spec.platform == Gem::Platform::RUBY
+      "#{package_name}.gem"
+    else
+      "#{package_name}-#{@gem_spec.platform}.gem"
+    end
+  end
+end  # class GemPackageTask
+end  # module Bones
+
+namespace :gem do
+
+  PROJ.gem._spec = Gem::Specification.new do |s|
+    s.name = PROJ.name
+    s.version = PROJ.version
+    s.summary = PROJ.summary
+    s.authors = Array(PROJ.authors)
+    s.email = PROJ.email
+    s.homepage = Array(PROJ.url).first
+    s.rubyforge_project = PROJ.rubyforge.name
+
+    s.description = PROJ.description
+
+    PROJ.gem.dependencies.each do |dep|
+      s.add_dependency(*dep)
+    end
+
+    PROJ.gem.development_dependencies.each do |dep|
+      s.add_development_dependency(*dep)
+    end
+
+    s.files = PROJ.gem.files
+    s.executables = PROJ.gem.executables.map {|fn| File.basename(fn)}
+    s.extensions = PROJ.gem.files.grep %r/extconf\.rb$/
+
+    s.bindir = 'bin'
+    dirs = Dir["{#{PROJ.libs.join(',')}}"]
+    s.require_paths = dirs unless dirs.empty?
+
+    incl = Regexp.new(PROJ.rdoc.include.join('|'))
+    excl = PROJ.rdoc.exclude.dup.concat %w[\.rb$ ^(\.\/|\/)?ext]
+    excl = Regexp.new(excl.join('|'))
+    rdoc_files = PROJ.gem.files.find_all do |fn|
+                   case fn
+                   when excl; false
+                   when incl; true
+                   else false end
+                 end
+    s.rdoc_options = PROJ.rdoc.opts + ['--main', PROJ.rdoc.main]
+    s.extra_rdoc_files = rdoc_files
+    s.has_rdoc = true
+
+    if test ?f, PROJ.test.file
+      s.test_file = PROJ.test.file
+    else
+      s.test_files = PROJ.test.files.to_a
+    end
+
+    # Do any extra stuff the user wants
+    PROJ.gem.extras.each do |msg, val|
+      case val
+      when Proc
+        val.call(s.send(msg))
+      else
+        s.send "#{msg}=", val
+      end
+    end
+  end  # Gem::Specification.new
+
+  Bones::GemPackageTask.new(PROJ.gem._spec) do |pkg|
+    pkg.need_tar = PROJ.gem.need_tar
+    pkg.need_zip = PROJ.gem.need_zip
+  end
+
+  desc 'Show information about the gem'
+  task :debug => 'gem:prereqs' do
+    puts PROJ.gem._spec.to_ruby
+  end
+
+  desc 'Write the gemspec '
+  task :spec => 'gem:prereqs' do
+    File.open("#{PROJ.name}.gemspec", 'w') do |f|
+      f.write PROJ.gem._spec.to_ruby
+    end
+  end
+
+  desc 'Install the gem'
+  task :install => [:clobber, 'gem:package'] do
+    sh "#{SUDO} #{GEM} install --local pkg/#{PROJ.gem._spec.full_name}"
+
+    # use this version of the command for rubygems > 1.0.0
+    #sh "#{SUDO} #{GEM} install --no-update-sources pkg/#{PROJ.gem._spec.full_name}"
+  end
+
+  desc 'Uninstall the gem'
+  task :uninstall do
+    installed_list = Gem.source_index.find_name(PROJ.name)
+    if installed_list and installed_list.collect { |s| s.version.to_s}.include?(PROJ.version) then
+      sh "#{SUDO} #{GEM} uninstall --version '#{PROJ.version}' --ignore-dependencies --executables #{PROJ.name}"
+    end
+  end
+
+  desc 'Reinstall the gem'
+  task :reinstall => [:uninstall, :install]
+
+  desc 'Cleanup the gem'
+  task :cleanup do
+    sh "#{SUDO} #{GEM} cleanup #{PROJ.gem._spec.name}"
+  end
+end  # namespace :gem
+
+
+desc 'Alias to gem:package'
+task :gem => 'gem:package'
+
+task :clobber => 'gem:clobber_package'
+remove_desc_for_task 'gem:clobber_package'
+
+# EOF