watir-schema 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +21 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +22 -0
- data/README.md +107 -0
- data/Rakefile +88 -0
- data/bin/watir-schema +64 -0
- data/lib/watir.rb +26 -0
- data/lib/watir/schema.rb +298 -0
- data/lib/watir/schema/version.rb +5 -0
- data/share/msie_err_control.ahk +22 -0
- data/share/settings.yaml +46 -0
- data/share/setup_linux +14 -0
- data/watir-schema.gemspec +28 -0
- metadata +118 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 5f035e29b54d53c796de2183ab54ee2396838c0a
|
4
|
+
data.tar.gz: 8ff51c485ac615d7312cd40f56e2279445fc0d70
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: a214469c05a4720e328ae3bfd9273c9768dc14dbe0ccba276a89d55a632bccdc59248320a2b3cfbb4f990b3aa564b9ec9e83771950bb89f3fedc19dd93dbbd48
|
7
|
+
data.tar.gz: 34a050e10ae61a462eb7612059d6ba2781246ddc882f092d1e5320ae2b2e155c6bd029f95619a0d2f18bef3272462881c88f87c54514004070040be0247d2132
|
data/.gitignore
ADDED
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2013 Malo Skrylevo
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,107 @@
|
|
1
|
+
# Watir::Schema
|
2
|
+
|
3
|
+
[](http://rubygems.org/gems/watir-schema)
|
4
|
+
[](https://travis-ci.org/3aHyga/watir-schema)
|
5
|
+
[](https://coveralls.io/r/3aHyga/watir-schema)
|
6
|
+
[](https://codeclimate.com/github/3aHyga/watir-schema)
|
7
|
+
[](http://coderwall.com/3aHyga)
|
8
|
+
|
9
|
+
|
10
|
+
Support multiple platforms (Windows, OS X, etc.) and browsers (Chrome, Firefox, Safari, IE, etc.):
|
11
|
+
- Chrome
|
12
|
+
- Firefox
|
13
|
+
- IE*
|
14
|
+
- Safari*
|
15
|
+
* - no proxy support
|
16
|
+
|
17
|
+
The tool is able to run around the clock with report at least every hour. Run of the script can be done with any scheduler like cron.
|
18
|
+
|
19
|
+
Functions:
|
20
|
+
- Launch a web surfer that will:
|
21
|
+
- Allow to use a proxy on 8080 port;
|
22
|
+
- Do a deep surf using the specified schema, including, for example, authenticated browsing on numerous websites, interacting with web forms, and random exploration of webpages;
|
23
|
+
- Return the browser's work folder;
|
24
|
+
- Store all of the surf data in a programmatically searchable log file immediately upon completion of the automated web surfing;
|
25
|
+
- Store the log results to the body of an email and send the results to the specified address at the end of each surf;
|
26
|
+
|
27
|
+
## Requirements
|
28
|
+
|
29
|
+
###Linux/MacOSX:
|
30
|
+
- chromedriver: http://code.google.com/p/selenium/wiki/ChromeDriver
|
31
|
+
|
32
|
+
###Windows:
|
33
|
+
- autohotkey.exe: http://www.autohotkey.com/
|
34
|
+
- IEDriverServer.exe: http://code.google.com/p/selenium/wiki/InternetExplorerDriver
|
35
|
+
- chromedriver.exe: http://code.google.com/p/selenium/wiki/ChromeDriver
|
36
|
+
All Programs must be included in PATH variable.
|
37
|
+
|
38
|
+
## Installation
|
39
|
+
|
40
|
+
Add this line to your application's Gemfile:
|
41
|
+
|
42
|
+
gem 'watir-schema'
|
43
|
+
|
44
|
+
And then execute:
|
45
|
+
|
46
|
+
$ bundle
|
47
|
+
|
48
|
+
Or install it yourself as:
|
49
|
+
|
50
|
+
$ gem install watir-schema
|
51
|
+
|
52
|
+
## Usage
|
53
|
+
|
54
|
+
Sample run:
|
55
|
+
$ watir-schema -v 5 -s schema.yaml -d ~/ -e sample@gmail.com
|
56
|
+
|
57
|
+
This will set log level 5 (extremely excessive), surf the web using schema.yaml config, and store browser data in the '~/' folder, and then send report to collected log specified sample@gmail.com.
|
58
|
+
|
59
|
+
Help info on the tool:
|
60
|
+
|
61
|
+
$ watir-schema -h
|
62
|
+
This is a watir-schema script, how to use it see below
|
63
|
+
-v, --verbose 1 enable verbose output, values: 0 to 5
|
64
|
+
-s, --schema set schema YAML-file to proceed
|
65
|
+
-b, --browser firefox use browser to surf, values: msie|firefox|chrome|safari
|
66
|
+
-p, --[no-]proxy use proxy
|
67
|
+
-d, --dir set base dir to store results
|
68
|
+
-t, --store store web-browser state into a folder
|
69
|
+
-e, --email set an email to report test results
|
70
|
+
-l, --log set output flow to a file
|
71
|
+
-h, --help Show this message
|
72
|
+
-V, --version Print version
|
73
|
+
|
74
|
+
## File schema.yaml
|
75
|
+
|
76
|
+
Sample schema.yaml is shewn below:
|
77
|
+
|
78
|
+
---
|
79
|
+
report:
|
80
|
+
server: smtp.gmail.com
|
81
|
+
port: 587
|
82
|
+
domain: networkadvertising.org
|
83
|
+
login: monitoring@networkadvertising.org
|
84
|
+
tls: true
|
85
|
+
password: '#rrrjjhdr@'
|
86
|
+
auth_type: :plain #or :plain or :login or :cram_md5
|
87
|
+
from: 'monitoring@networkadvertising.org'
|
88
|
+
from_name: 'NAI Monitoring'
|
89
|
+
body: ! "From: %from_name <%from>\n
|
90
|
+
Subject: Watir web-surfing report"
|
91
|
+
sites:
|
92
|
+
google.com:
|
93
|
+
schema:
|
94
|
+
- 'a:id:gb_70:%C'
|
95
|
+
- 't:id:Email:%S=monitoring@networkadvertising.org='
|
96
|
+
- 't:id:Passwd:%S=#rrrjjhdr@='
|
97
|
+
- 'i:id:signIn:C'
|
98
|
+
- 't:name:q:%S=2012 Audi A6='
|
99
|
+
- 'b:name:btnG:C'
|
100
|
+
|
101
|
+
## Contributing
|
102
|
+
|
103
|
+
1. Fork it
|
104
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
105
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
106
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
107
|
+
5. Create new Pull Request
|
data/Rakefile
ADDED
@@ -0,0 +1,88 @@
|
|
1
|
+
#!/usr/bin/env rake
|
2
|
+
|
3
|
+
desc "Prepare bundler"
|
4
|
+
task :bundleup do
|
5
|
+
sh 'gem install bundler --version "~> 1.3.1" --no-ri --no-rdoc'
|
6
|
+
end
|
7
|
+
|
8
|
+
desc "Requires"
|
9
|
+
task :req do
|
10
|
+
$: << File.expand_path( '../lib', __FILE__ )
|
11
|
+
require 'bundler/gem_helper'
|
12
|
+
require 'watir'
|
13
|
+
|
14
|
+
Bundler::GemHelper.install_tasks
|
15
|
+
end
|
16
|
+
|
17
|
+
desc "Prepare bundle environment"
|
18
|
+
task :up do
|
19
|
+
sh 'bundle install'
|
20
|
+
end
|
21
|
+
|
22
|
+
desc "Test with cucumber"
|
23
|
+
task :test do
|
24
|
+
sh 'if [ -d features ]; then tests=$(ls features/*.feature) ; cucumber $tests; fi'
|
25
|
+
end
|
26
|
+
|
27
|
+
desc "Distilled clean"
|
28
|
+
task :distclean do
|
29
|
+
sh 'git clean -fd'
|
30
|
+
sh 'cat .gitignore | while read mask; do rm -rf $(find -iname "$mask"); done'
|
31
|
+
end
|
32
|
+
|
33
|
+
desc "Prepare environment"
|
34
|
+
task :env => [ :req ] do
|
35
|
+
files = [ 'http://selenium.googlecode.com/files/selenium-server-2.31.0.zip' ]
|
36
|
+
drivers = case RUBY_PLATFORM
|
37
|
+
when /linux/
|
38
|
+
sh "#{Watir::Schema.root}/share/setup_linux #{Watir::Schema.root}"
|
39
|
+
[ RUBY_PLATFORM =~ /64/ && 'chromedriver_linux64_26.0.1383.0.zip' ||
|
40
|
+
'chromedriver_linux32_26.0.1383.0.zip' ]
|
41
|
+
when /windows/
|
42
|
+
[ RUBY_PLATFORM =~ /64/ && 'IEDriverServer_x64_2.31.0.zip' ||
|
43
|
+
'IEDriverServer_Win32_2.31.0.zip',
|
44
|
+
'chromedriver_win_26.0.1383.0.zip' ]
|
45
|
+
when /macosx/
|
46
|
+
[ 'chromedriver_mac_26.0.1383.0.zip' ]
|
47
|
+
else [] ; end
|
48
|
+
drivers.each do |driver|
|
49
|
+
files << "http://chromedriver.googlecode.com/files/#{driver}" ; end
|
50
|
+
|
51
|
+
files.each do |file|
|
52
|
+
filename = File.basename file
|
53
|
+
target = File.join Watir::Schema.root, filename
|
54
|
+
sh "wget #{file} -O #{target}"
|
55
|
+
sh "7z x #{target}"
|
56
|
+
end
|
57
|
+
|
58
|
+
puts "-" * 30 + "\nPlease, reload your environment"
|
59
|
+
end
|
60
|
+
|
61
|
+
desc "Generate gem"
|
62
|
+
namespace :gem do
|
63
|
+
task :build => [ :req ] do
|
64
|
+
sh 'gem build watir-schema.gemspec'
|
65
|
+
end
|
66
|
+
|
67
|
+
task :install do
|
68
|
+
require File.expand_path( '../lib/watir/schema/version', __FILE__ )
|
69
|
+
sh "gem install watir-schema-#{Watir::Schema::VERSION}.gem"
|
70
|
+
end
|
71
|
+
|
72
|
+
task :publish => [ :req ] do
|
73
|
+
require File.expand_path( '../lib/watir/schema/version', __FILE__ )
|
74
|
+
sh "git tag v#{Watir::Schema::VERSION}"
|
75
|
+
sh "gem push watir-schema-#{Watir::Schema::VERSION}.gem"
|
76
|
+
sh "git push"
|
77
|
+
sh "git push --tag"
|
78
|
+
end
|
79
|
+
|
80
|
+
task :make => [ :build, :install, :publish ]
|
81
|
+
task :default => :make
|
82
|
+
end
|
83
|
+
|
84
|
+
task(:default).clear
|
85
|
+
task :default => :test
|
86
|
+
task :all => [ :bundleup, :up, :test, :'gem:make', :distclean ]
|
87
|
+
task :build => [ :bundleup, :up, :test, :'gem:build', :'gem:install', :distclean ]
|
88
|
+
|
data/bin/watir-schema
ADDED
@@ -0,0 +1,64 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'micro-optparse'
|
4
|
+
require 'yaml'
|
5
|
+
require 'watir/schema'
|
6
|
+
|
7
|
+
options = Parser.new do |p|
|
8
|
+
p.banner = "This is a watir-schema script, how to use it see below"
|
9
|
+
p.version = "watir-schema #{Watir::Schema::VERSION}"
|
10
|
+
p.option :verbose, "enable verbose output, values: 0 to 5",
|
11
|
+
:short => 'v', :default => 1, :value_in_set => ( 0..5 ).to_a
|
12
|
+
p.option :schema, "set schema YAML-file to proceed",
|
13
|
+
:short => 's', :default => '', :value_matches => /^.+$/
|
14
|
+
p.option :browser, "use browser to surf, values: " +
|
15
|
+
"msie|firefox|chrome|safari",
|
16
|
+
:short => 'b', :default => 'firefox',
|
17
|
+
:value_matches => /^(msie|firefox|chrome|safari)$/
|
18
|
+
p.option :proxy, "use proxy", :short => 'p'
|
19
|
+
p.option :dir, "set base dir to store results",
|
20
|
+
:short => 'd', :default => ''
|
21
|
+
p.option :store, "store web-browser state into a folder",
|
22
|
+
:short => 't', :default => ''
|
23
|
+
p.option :email, "set an email to report test results",
|
24
|
+
:short => 'e', :default => '',
|
25
|
+
:value_satisfies => lambda {|x| x.empty? || x =~ /[^@]+@[^@]+/ }
|
26
|
+
p.option :log, "set output flow to a file", :short => 'l', :default => ''
|
27
|
+
end.process!
|
28
|
+
if !ARGV.empty?
|
29
|
+
$stderr.puts "Invalid option(s): #{ARGV.inspect}" ; exit ; end
|
30
|
+
|
31
|
+
f = case options[ :verbose ]
|
32
|
+
when 0
|
33
|
+
[ :warn ]
|
34
|
+
when 1
|
35
|
+
[ :info, :warn ]
|
36
|
+
when 2
|
37
|
+
[ :info, :warn, :basic ]
|
38
|
+
when 3
|
39
|
+
[ :info, :warn, :basic, :extended ]
|
40
|
+
when 4
|
41
|
+
[ :info, :warn, :basic, :enter, :leave ]
|
42
|
+
when 5
|
43
|
+
[ :info, :warn, :basic, :extended, :enter, :leave ] ; end
|
44
|
+
|
45
|
+
io = ( options[ :log ] == '-' || !options[ :email ].empty? ) &&
|
46
|
+
( require 'tempfile' ; Tempfile.new( 'tmp.log' ) ) ||
|
47
|
+
options[ :log ].empty? && $stdout ||
|
48
|
+
File.open( options[ :log ], 'w+' )
|
49
|
+
|
50
|
+
require 'rdoba'
|
51
|
+
settings = YAML::load( open Watir::Schema.root( 'share/settings.yaml' ) )
|
52
|
+
|
53
|
+
$stdout.puts "The script is running..."
|
54
|
+
|
55
|
+
rdoba :log => { :in => Watir::Schema, :as => :log, :prefix => [ :timestamp ],
|
56
|
+
:io => io, :functions => f }
|
57
|
+
|
58
|
+
surfer = Watir::Schema.new settings, options[ :browser ],
|
59
|
+
options[ :dir ]
|
60
|
+
|
61
|
+
schema = YAML::load( open options[ :schema ] )
|
62
|
+
surfer.crawl schema, options[ :proxy ], options[ :store ]
|
63
|
+
surfer.report schema, options[ :email ], io, options[ :log ]
|
64
|
+
|
data/lib/watir.rb
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
require 'rbconfig'
|
2
|
+
require 'watir/schema'
|
3
|
+
require 'watir/schema/version'
|
4
|
+
|
5
|
+
module Watir
|
6
|
+
def self.os
|
7
|
+
@@os ||= (
|
8
|
+
host_os = RbConfig::CONFIG['host_os']
|
9
|
+
case host_os
|
10
|
+
when /(mswin|msys|mingw|cygwin|bccwin|wince|emc)/
|
11
|
+
plat = $1 == 'mswin' && 'native' || $1
|
12
|
+
out = `ver`.encode( 'US-ASCII',
|
13
|
+
:invalid => :replace, :undef => :replace )
|
14
|
+
if out =~ /\[.* (\d+)\.([\d\.]+)\]/
|
15
|
+
"windows-#{plat}-#{$1 == '5' && 'xp' || 'vista'}-#{$1}.#{$2}"
|
16
|
+
else
|
17
|
+
"windows-#{plat}" ; end
|
18
|
+
when /darwin|mac os/
|
19
|
+
'macosx'
|
20
|
+
when /linux/
|
21
|
+
'linux'
|
22
|
+
when /(solaris|bsd)/
|
23
|
+
"unix-#{$1}"
|
24
|
+
else
|
25
|
+
raise Error::WebDriverError, "unknown os: #{host_os.inspect}"
|
26
|
+
end) ; end ; end
|
data/lib/watir/schema.rb
ADDED
@@ -0,0 +1,298 @@
|
|
1
|
+
require 'watir'
|
2
|
+
require 'watir-webdriver'
|
3
|
+
require 'fileutils'
|
4
|
+
require 'open3'
|
5
|
+
require 'yaml'
|
6
|
+
require 'timeout'
|
7
|
+
require 'net/smtp'
|
8
|
+
|
9
|
+
module Watir
|
10
|
+
class Browser
|
11
|
+
module Firefox
|
12
|
+
def browser_run options
|
13
|
+
b = Watir::Browser.new :firefox, options
|
14
|
+
profile_dir = b.driver.send( :bridge ).
|
15
|
+
instance_variable_get( :@launcher ).
|
16
|
+
instance_variable_get( :@profile_dir )
|
17
|
+
[ b, [ profile_dir ].freeze ] ; end
|
18
|
+
|
19
|
+
def browser_init browser, proxy = false
|
20
|
+
profile = Selenium::WebDriver::Firefox::Profile.new
|
21
|
+
profile.native_events = true
|
22
|
+
@options = { :profile => profile }
|
23
|
+
if proxy
|
24
|
+
@options.merge :proxy => { :http => "http://localhost:8080" } ; end
|
25
|
+
|
26
|
+
@b, @profile_data = browser_run @options ; end ; end
|
27
|
+
|
28
|
+
module Chrome
|
29
|
+
def browser_init browser, proxy = false
|
30
|
+
@profile_data = [ File.join( @path, 'profile' ) ].freeze
|
31
|
+
switches = [ "--user-data-dir=#{@profile_data.last}" ]
|
32
|
+
if proxy
|
33
|
+
switches << "--proxy-server=localhost:8080" ; end
|
34
|
+
@options = { :switches => switches }
|
35
|
+
@b = Watir::Browser.new :chrome, @options ; end ; end
|
36
|
+
|
37
|
+
module Msie
|
38
|
+
def browser_init browser, proxy = false
|
39
|
+
clear_current_state
|
40
|
+
# MSIE has no plain proxy settings
|
41
|
+
si, so, se, @wthr = Open3.popen3( "AutoHotkey " +
|
42
|
+
"\"#{fix_path Watir::Schema.root( "share/msie_err_control.ahk" )}\"" )
|
43
|
+
@b = Watir::Browser.new :ie ; end ; end
|
44
|
+
|
45
|
+
module Safari
|
46
|
+
def browser_init browser, proxy = false
|
47
|
+
clear_current_state
|
48
|
+
# Safari has no plain proxy settings
|
49
|
+
@b = Watir::Browser.new :safari
|
50
|
+
end ; end
|
51
|
+
|
52
|
+
module Opera
|
53
|
+
def browser_init browser, proxy = false
|
54
|
+
# http://code.google.com/p/selenium/wiki/OperaDriver
|
55
|
+
# Opera driver have no proxy caps
|
56
|
+
@profile_data = [ File.join( @path, 'profile' ) ].freeze
|
57
|
+
@b = Watir::Browser.new :opera, { :profile => @profile_data.last }
|
58
|
+
end ; end ; end
|
59
|
+
|
60
|
+
class Schema
|
61
|
+
Actions = {
|
62
|
+
's' => '.select_list',
|
63
|
+
'o' => '.option',
|
64
|
+
't' => '.text_field',
|
65
|
+
'a' => '.a',
|
66
|
+
'i' => '.input',
|
67
|
+
'b' => '.button',
|
68
|
+
'c' => '.checkbox',
|
69
|
+
'f' => '.form',
|
70
|
+
'd' => '.div',
|
71
|
+
'p' => '.span',
|
72
|
+
'e' => '.element',
|
73
|
+
'%' => '.when_present',
|
74
|
+
'L' => '.select',
|
75
|
+
'S' => '.set',
|
76
|
+
'C' => '.click',
|
77
|
+
'T' => '.text',
|
78
|
+
}
|
79
|
+
AKeys = Actions.keys.join().gsub(/[?]/) do |e| "\\#{e}" ; end
|
80
|
+
|
81
|
+
def self.root path = ''
|
82
|
+
root = File.expand_path '../../..', __FILE__
|
83
|
+
File.join root, path
|
84
|
+
end
|
85
|
+
|
86
|
+
def initialize settings, browser = 'firefox', basedir = nil
|
87
|
+
# Initialize browser
|
88
|
+
@browser = browser.to_sym
|
89
|
+
if ![ :chrome, :firefox, :msie, :opera, :safari ].include? @browser
|
90
|
+
raise "#{@browser.inspect} isn't supported" ; end
|
91
|
+
@browser_settings = settings[ 'browsers' ][ @browser.to_s ]
|
92
|
+
eval "extend Browser::#{@browser.capitalize}"
|
93
|
+
log * { Browser: browser }
|
94
|
+
|
95
|
+
# Initialize paths for data logs
|
96
|
+
basedir = !basedir.empty? && basedir || ENV[ 'TMP' ]
|
97
|
+
@path = File.join( basedir, 'watir-scheme', ::Time.now.strftime('%m-%d-%Y'),
|
98
|
+
"#{browser.capitalize}-#{::Time.now.strftime("%H_%M_%S")}" )
|
99
|
+
log * { Path: @path }
|
100
|
+
FileUtils.mkdir_p @path
|
101
|
+
|
102
|
+
# Initialize timeouts
|
103
|
+
@load_timeout = @browser_settings[ 'timeouts' ] &&
|
104
|
+
@browser_settings[ 'timeouts' ][ 'load' ] || settings[ :timeouts ] &&
|
105
|
+
settings[ :timeouts ][ :load ] || 60
|
106
|
+
@automate_timeout = @browser_settings[ 'timeouts' ] &&
|
107
|
+
@browser_settings[ 'timeouts' ][ 'automate' ] ||
|
108
|
+
settings[ :timeouts ] && settings[ :timeouts ][ :automate ] || 600
|
109
|
+
|
110
|
+
# Data folders to store
|
111
|
+
( @browser_settings[ 'data' ] || {} ).each do |os, data|
|
112
|
+
if Watir.os =~ /#{os}/
|
113
|
+
@profile_data = data ; end ; end
|
114
|
+
|
115
|
+
# Store settings
|
116
|
+
log >> { Settings: settings }
|
117
|
+
@settings = settings ; end
|
118
|
+
|
119
|
+
def crawl schema, proxy, storedir
|
120
|
+
log + { schema: schema, proxy: proxy, storedir: storedir }
|
121
|
+
|
122
|
+
#Initialize testing variables
|
123
|
+
pid = nil
|
124
|
+
|
125
|
+
#Initialize browser
|
126
|
+
browser_init @browser,
|
127
|
+
proxy && ( schema[ :proxy ] || settings[ :proxy ] )
|
128
|
+
|
129
|
+
#Begin web crawl
|
130
|
+
surf schema[ 'sites' ] || []
|
131
|
+
if storedir && !storedir.empty?
|
132
|
+
store_state storedir ; end
|
133
|
+
ensure
|
134
|
+
if @b
|
135
|
+
# close browser
|
136
|
+
@b.close rescue nil ; end ; end
|
137
|
+
|
138
|
+
def report schema, email, io, l = nil
|
139
|
+
if io == $stdout || !email || email.empty?
|
140
|
+
return ; end
|
141
|
+
|
142
|
+
rep = schema[ 'report' ]
|
143
|
+
status = ''
|
144
|
+
smtp = Net::SMTP.new rep[ 'server' ], rep[ 'port' ]
|
145
|
+
if rep[ 'tls' ]
|
146
|
+
smtp.enable_starttls ; end
|
147
|
+
io.rewind
|
148
|
+
begin
|
149
|
+
smtp.start( rep[ 'domain' ],
|
150
|
+
rep[ 'login' ], rep[ 'password' ], rep[ 'auth_type' ] ) do |s|
|
151
|
+
msg = rep[ 'body' ].gsub( /%[_\w]+/ ) do |str|
|
152
|
+
rep[ str.sub( '%', '' ) ] ; end.split( "\n" ).map do |x|
|
153
|
+
x.strip end.join( "\n")
|
154
|
+
msg << "\nTo: #{email}\n\nThe log is below:\n\n"
|
155
|
+
msg << io.read
|
156
|
+
s.send_message msg, rep[ 'from' ], email
|
157
|
+
end
|
158
|
+
status = "Report has been sent"
|
159
|
+
rescue
|
160
|
+
log.e
|
161
|
+
status = "It is unable to send the log"
|
162
|
+
l = true ; end
|
163
|
+
|
164
|
+
if l && !l.empty?
|
165
|
+
io.rewind
|
166
|
+
$stdout.puts io.read ; end
|
167
|
+
$stdout.puts status ; end
|
168
|
+
|
169
|
+
private
|
170
|
+
|
171
|
+
def fix_path path
|
172
|
+
if Watir.os =~ /windows/
|
173
|
+
path.gsub!( /\//, "\\" ) ; end
|
174
|
+
path ; end
|
175
|
+
|
176
|
+
def is_fullpath? path
|
177
|
+
if Watir.os =~ /windows/
|
178
|
+
path[ 1 ] == ':' && ( path[ 2 ] == "\\" || path[ 2 ] == '/' )
|
179
|
+
else
|
180
|
+
path[ 0 ] == '/' ; end ; end
|
181
|
+
|
182
|
+
def store_state folder = nil
|
183
|
+
log + { folder: folder }
|
184
|
+
target_base = folder
|
185
|
+
target_dir = target_base
|
186
|
+
log > "Storing browser state to #{target_dir}"
|
187
|
+
home = Dir.home
|
188
|
+
init = if @profile_data.is_a? Hash
|
189
|
+
Proc.new do |dir|
|
190
|
+
source = dir[ 1 ]
|
191
|
+
Dir.chdir( is_fullpath?( source ) && source ||
|
192
|
+
File.join( home, source ) )
|
193
|
+
target_dir = File.join target_base, dir[ 0 ] ; end
|
194
|
+
else
|
195
|
+
Proc.new do |dir|
|
196
|
+
Dir.chdir( is_fullpath?( dir ) && dir ||
|
197
|
+
File.join( home, dir ) ) ; end ; end
|
198
|
+
|
199
|
+
@profile_data.each do |dir|
|
200
|
+
init.call dir
|
201
|
+
Dir.glob( File.join '**', '*' ).each do |file|
|
202
|
+
tfile = File.join( target_dir, file )
|
203
|
+
FileUtils.mkdir_p( File.dirname tfile )
|
204
|
+
FileUtils.cp_r( file, tfile ) rescue Exception
|
205
|
+
end ; end
|
206
|
+
|
207
|
+
clear_current_state ; end
|
208
|
+
|
209
|
+
def clear_current_state
|
210
|
+
if @profile_data.frozen?
|
211
|
+
return ; end
|
212
|
+
|
213
|
+
home = Dir.home
|
214
|
+
init = if @profile_data.is_a? Hash
|
215
|
+
Proc.new do |dir|
|
216
|
+
source = dir[ 1 ]
|
217
|
+
Dir.chdir( is_fullpath?( source ) && source ||
|
218
|
+
File.join( home, source ) ) ; end
|
219
|
+
else
|
220
|
+
Proc.new do |dir|
|
221
|
+
Dir.chdir( is_fullpath( dir ) && dir ||
|
222
|
+
File.join( home, dir ) ) ; end ; end
|
223
|
+
|
224
|
+
@profile_data.each do |dir|
|
225
|
+
init.call dir
|
226
|
+
list = Dir.glob '*'
|
227
|
+
list.delete 'index.dat'
|
228
|
+
FileUtils.rm_rf list rescue Exception ; end ; end
|
229
|
+
|
230
|
+
def surf sites
|
231
|
+
log + { :sites => sites }
|
232
|
+
b = @b
|
233
|
+
log >> { :b => b }
|
234
|
+
sites.each do |site, opts|
|
235
|
+
log > { Site: site }
|
236
|
+
log >> { Opts: opts }
|
237
|
+
|
238
|
+
uri = site + ( opts[ 'path' ] || '' )
|
239
|
+
log > { uri: uri }
|
240
|
+
begin
|
241
|
+
Timeout::timeout( @load_timeout ) do
|
242
|
+
b.goto uri ; end
|
243
|
+
rescue Timeout::Error
|
244
|
+
log % "Error: can't complete the site #{uri} surfing"
|
245
|
+
next ; end
|
246
|
+
|
247
|
+
schema = opts[ 'schema' ]
|
248
|
+
log > "#{schema.size} steps in the schema for the site have found"
|
249
|
+
schema.each do |act|
|
250
|
+
log >> { act: act }
|
251
|
+
e = 'b'
|
252
|
+
while act.size > 0
|
253
|
+
if act =~ /^(?:([@?#{AKeys}])|(:([\w\-]+):([^:]+):)|([\(=`~](.*[^\\\)=`~])[\)=`~]))/
|
254
|
+
log >> { Matches: [ $1, $2, $3, $4, $5, $6 ] }
|
255
|
+
if $1
|
256
|
+
case $1
|
257
|
+
when '@'
|
258
|
+
e = "Watir::Wait.until(@automate_timeout) {(#{e})}"
|
259
|
+
when '?'
|
260
|
+
e = "(#{e}) if b"
|
261
|
+
else
|
262
|
+
e << Actions[ $1 ] ; end
|
263
|
+
act = act[ 1..-1 ]
|
264
|
+
elsif $2
|
265
|
+
e << "(:'#{$3}'=>'#{$4}')"
|
266
|
+
act = act[ $2.size..-1 ]
|
267
|
+
elsif $5
|
268
|
+
match = $5
|
269
|
+
word = $6.gsub( /\\(.)/, '\1' )
|
270
|
+
case match[ 0 ]
|
271
|
+
when '~'
|
272
|
+
e << " =~ /#{word}/"
|
273
|
+
when '='
|
274
|
+
e << "('#{word}')"
|
275
|
+
when '('
|
276
|
+
e = "sleep (#{word}); #{e}"
|
277
|
+
else
|
278
|
+
e << " !~ /#{word}/" ; end
|
279
|
+
act = act[ match.size..-1 ] ; end
|
280
|
+
else
|
281
|
+
log.e "Unknown operation #{act[ 0 ]} in schema part: #{act}"
|
282
|
+
act = act[ 1..-1 ] ; end ; end
|
283
|
+
log > "Step: eval #{e}"
|
284
|
+
begin
|
285
|
+
Timeout::timeout( @automate_timeout ) do
|
286
|
+
eval e ; end
|
287
|
+
rescue Timeout::Error
|
288
|
+
log > "Error: We didn't wait out the Opt-out analyzing completed"
|
289
|
+
rescue Watir::Wait::TimeoutError;
|
290
|
+
log > "Error: Can't execute #{e}"
|
291
|
+
rescue Interrupt
|
292
|
+
return nil
|
293
|
+
rescue Exception
|
294
|
+
log.e
|
295
|
+
end ; end ; end
|
296
|
+
|
297
|
+
true ; end ; end ; end
|
298
|
+
|
@@ -0,0 +1,22 @@
|
|
1
|
+
#ErrorStdOut
|
2
|
+
SetTitleMatchMode, RegEx
|
3
|
+
SetControlDelay, 1000
|
4
|
+
|
5
|
+
WinWait, ahk_class IEFrame
|
6
|
+
WinActivate
|
7
|
+
Loop,
|
8
|
+
{
|
9
|
+
WinWaitActive, ahk_class #32770,,5
|
10
|
+
if ! ErrorLevel
|
11
|
+
Loop,
|
12
|
+
{
|
13
|
+
ControlClick, Button1 ; OK
|
14
|
+
WinWaitClose, ahk_class #32770,,1
|
15
|
+
if ! ErrorLevel
|
16
|
+
Break
|
17
|
+
}
|
18
|
+
|
19
|
+
WinWaitClose, ahk_class IEFrame,,1
|
20
|
+
if ! ErrorLevel
|
21
|
+
Break
|
22
|
+
}
|
data/share/settings.yaml
ADDED
@@ -0,0 +1,46 @@
|
|
1
|
+
---
|
2
|
+
default_browser: firefox
|
3
|
+
optouts: true
|
4
|
+
proxy:
|
5
|
+
run: false
|
6
|
+
port: 8080
|
7
|
+
timeouts:
|
8
|
+
load: 120
|
9
|
+
automate: 1200
|
10
|
+
browsers:
|
11
|
+
firefox:
|
12
|
+
timeouts:
|
13
|
+
load: 60
|
14
|
+
automate: 600
|
15
|
+
chrome:
|
16
|
+
timeouts:
|
17
|
+
load: 60
|
18
|
+
automate: 300
|
19
|
+
msie:
|
20
|
+
timeouts:
|
21
|
+
load: 60
|
22
|
+
automate: 600
|
23
|
+
data:
|
24
|
+
xp:
|
25
|
+
Cookies: Cookies
|
26
|
+
Cache: Local Settings/Temporary Internet Files/Content.IE5
|
27
|
+
History: Local Settings/History/History.IE5
|
28
|
+
vista:
|
29
|
+
Cookies: AppData/Roaming/Microsoft/Windows/Cookies
|
30
|
+
Content: AppData/Local/Microsoft/Windows/Temporary Internet Files/Content.IE5
|
31
|
+
History: AppData/Local/Microsoft/Windows/History
|
32
|
+
safari:
|
33
|
+
timeouts:
|
34
|
+
load: 30
|
35
|
+
automate: 300
|
36
|
+
data:
|
37
|
+
xp:
|
38
|
+
- Local Settings/Application Data/Apple Computer/Safari
|
39
|
+
- Application Data/Apple Computer/Safari
|
40
|
+
vista:
|
41
|
+
- AppData/Roaming/Apple Computer/Safari
|
42
|
+
- AppData/Local/Apple Computer/Safari
|
43
|
+
ios:
|
44
|
+
- /private/var/mobile/Library
|
45
|
+
macosx:
|
46
|
+
- Library
|
data/share/setup_linux
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
#!/bin/bash
|
2
|
+
|
3
|
+
root=$1
|
4
|
+
|
5
|
+
env=$(cat ~/.bashrc | grep "SELENIUM_SERVER_JAR.*selenium-server-2.31.0.jar")
|
6
|
+
if [ -z "$env" ]; then
|
7
|
+
echo "export SELENIUM_SERVER_JAR=$root/selenium-2.31.0/selenium-2.31.0/selenium-server-2.31.0.jar" >> ~/.bashrc
|
8
|
+
fi
|
9
|
+
|
10
|
+
env=$(cat ~/.bashrc | grep "chromedriver")
|
11
|
+
if [ -z "$env" ]; then
|
12
|
+
echo "export PATH=\$PATH:$root" >> ~/.bashrc
|
13
|
+
fi
|
14
|
+
|
@@ -0,0 +1,28 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'watir/schema/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "watir-schema"
|
8
|
+
spec.version = Watir::Schema::VERSION
|
9
|
+
spec.authors = ["Malo Skrylevo"]
|
10
|
+
spec.email = ["majioa@yandex.ru"]
|
11
|
+
spec.description = %q{The gem is able to run around the clock with report at least every hour. Run of the script can be done with any scheduler like cron}
|
12
|
+
spec.summary = %q{The gem allows automated surfing over web using watir-webdriver}
|
13
|
+
spec.homepage = "https://github.com/3aHyga/watir-schema"
|
14
|
+
spec.license = "MIT"
|
15
|
+
|
16
|
+
spec.files = `git ls-files`.split($/)
|
17
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
18
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
19
|
+
spec.require_paths = ["lib"]
|
20
|
+
|
21
|
+
spec.add_dependency 'bundler', '~> 1.3.1'
|
22
|
+
spec.add_dependency 'watir-webdriver', '~> 0.6.2'
|
23
|
+
spec.add_dependency 'micro-optparse'
|
24
|
+
spec.add_dependency 'rdoba'
|
25
|
+
|
26
|
+
spec.add_development_dependency "bundler", "~> 1.3"
|
27
|
+
spec.add_development_dependency "rake"
|
28
|
+
end
|
metadata
ADDED
@@ -0,0 +1,118 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: watir-schema
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.2
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Malo Skrylevo
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
|
12
|
+
date: 2013-10-01 00:00:00 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: bundler
|
16
|
+
requirement: &id001 !ruby/object:Gem::Requirement
|
17
|
+
requirements:
|
18
|
+
- - ~>
|
19
|
+
- !ruby/object:Gem::Version
|
20
|
+
version: 1.3.1
|
21
|
+
type: :runtime
|
22
|
+
prerelease: false
|
23
|
+
version_requirements: *id001
|
24
|
+
- !ruby/object:Gem::Dependency
|
25
|
+
name: watir-webdriver
|
26
|
+
requirement: &id002 !ruby/object:Gem::Requirement
|
27
|
+
requirements:
|
28
|
+
- - ~>
|
29
|
+
- !ruby/object:Gem::Version
|
30
|
+
version: 0.6.2
|
31
|
+
type: :runtime
|
32
|
+
prerelease: false
|
33
|
+
version_requirements: *id002
|
34
|
+
- !ruby/object:Gem::Dependency
|
35
|
+
name: micro-optparse
|
36
|
+
requirement: &id003 !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- &id004
|
39
|
+
- ">="
|
40
|
+
- !ruby/object:Gem::Version
|
41
|
+
version: "0"
|
42
|
+
type: :runtime
|
43
|
+
prerelease: false
|
44
|
+
version_requirements: *id003
|
45
|
+
- !ruby/object:Gem::Dependency
|
46
|
+
name: rdoba
|
47
|
+
requirement: &id005 !ruby/object:Gem::Requirement
|
48
|
+
requirements:
|
49
|
+
- *id004
|
50
|
+
type: :runtime
|
51
|
+
prerelease: false
|
52
|
+
version_requirements: *id005
|
53
|
+
- !ruby/object:Gem::Dependency
|
54
|
+
name: bundler
|
55
|
+
requirement: &id006 !ruby/object:Gem::Requirement
|
56
|
+
requirements:
|
57
|
+
- - ~>
|
58
|
+
- !ruby/object:Gem::Version
|
59
|
+
version: "1.3"
|
60
|
+
type: :development
|
61
|
+
prerelease: false
|
62
|
+
version_requirements: *id006
|
63
|
+
- !ruby/object:Gem::Dependency
|
64
|
+
name: rake
|
65
|
+
requirement: &id007 !ruby/object:Gem::Requirement
|
66
|
+
requirements:
|
67
|
+
- *id004
|
68
|
+
type: :development
|
69
|
+
prerelease: false
|
70
|
+
version_requirements: *id007
|
71
|
+
description: The gem is able to run around the clock with report at least every hour. Run of the script can be done with any scheduler like cron
|
72
|
+
email:
|
73
|
+
- majioa@yandex.ru
|
74
|
+
executables:
|
75
|
+
- watir-schema
|
76
|
+
extensions: []
|
77
|
+
|
78
|
+
extra_rdoc_files: []
|
79
|
+
|
80
|
+
files:
|
81
|
+
- .gitignore
|
82
|
+
- Gemfile
|
83
|
+
- LICENSE.txt
|
84
|
+
- README.md
|
85
|
+
- Rakefile
|
86
|
+
- bin/watir-schema
|
87
|
+
- lib/watir.rb
|
88
|
+
- lib/watir/schema.rb
|
89
|
+
- lib/watir/schema/version.rb
|
90
|
+
- share/msie_err_control.ahk
|
91
|
+
- share/settings.yaml
|
92
|
+
- share/setup_linux
|
93
|
+
- watir-schema.gemspec
|
94
|
+
homepage: https://github.com/3aHyga/watir-schema
|
95
|
+
licenses:
|
96
|
+
- MIT
|
97
|
+
metadata: {}
|
98
|
+
|
99
|
+
post_install_message:
|
100
|
+
rdoc_options: []
|
101
|
+
|
102
|
+
require_paths:
|
103
|
+
- lib
|
104
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
105
|
+
requirements:
|
106
|
+
- *id004
|
107
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
108
|
+
requirements:
|
109
|
+
- *id004
|
110
|
+
requirements: []
|
111
|
+
|
112
|
+
rubyforge_project:
|
113
|
+
rubygems_version: 2.1.4
|
114
|
+
signing_key:
|
115
|
+
specification_version: 4
|
116
|
+
summary: The gem allows automated surfing over web using watir-webdriver
|
117
|
+
test_files: []
|
118
|
+
|