outrider 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +17 -0
  3. data/.rspec +2 -0
  4. data/.ruby-version +1 -0
  5. data/.travis.yml +3 -0
  6. data/Capfile +30 -0
  7. data/Gemfile +20 -0
  8. data/Gemfile.lock +119 -0
  9. data/LICENSE.txt +21 -0
  10. data/README.md +261 -0
  11. data/Rakefile +52 -0
  12. data/app/run.rb +15 -0
  13. data/bin/console +14 -0
  14. data/bin/outrider +8 -0
  15. data/bin/setup +7 -0
  16. data/config.ru +2 -0
  17. data/config/messages.yml +1 -0
  18. data/config/schema.sql +40 -0
  19. data/lib/ignite.rb +8 -0
  20. data/lib/outrider.rb +94 -0
  21. data/lib/outrider/commandify.rb +49 -0
  22. data/lib/outrider/engine.rb +20 -0
  23. data/lib/outrider/intel.rb +14 -0
  24. data/lib/outrider/project.rb +146 -0
  25. data/lib/outrider/tools.rb +224 -0
  26. data/lib/outrider/version.rb +3 -0
  27. data/outrider.gemspec +27 -0
  28. data/projects/nz_herald/auxiliary.rb +56 -0
  29. data/projects/stuff/auxiliary.rb +71 -0
  30. data/projects/test_project/auxiliary.rb +63 -0
  31. data/projects/theage/auxiliary.rb +29 -0
  32. data/public/index.html +0 -0
  33. data/tmp/x.txt +1 -0
  34. metadata +122 -0
data/projects/theage/auxiliary.rb ADDED
@@ -0,0 +1,29 @@
1
+ class Theage < Project
2
+ def initialize
3
+ project_name :theage
4
+ end
5
+
6
+
7
+ def crawl options
8
+ p OutriderTools::Crawl::site( @config, ->(page, uri){
9
+
10
+ unless( page.css('.cN-headingPage').text.strip.empty? )
11
+ clean_date = DateTime.strptime(page.css('.dtstamp time').text.strip, '%B %d, %Y').to_s #Tue Mar 03 08:27:23 UTC 2015
12
+ return {
13
+ :title_raw => page.css('h1.cN-headingPage').text.strip,
14
+ :author => page.css('.authorName a').text.strip,
15
+ :content_raw => page.css('.articleBody p').map{ |paragraph| paragraph.text.strip }.to_json,
16
+ :date_published_raw => page.css('.dtstamp time').text.strip,
17
+ :date_published_timestamp => clean_date,
18
+ :status => 'scraped'
19
+ }
20
+ else
21
+ return {
22
+ :status => 'rejected'
23
+ }
24
+ end
25
+ })
26
+ end
27
+
28
+
29
+ end
data/public/index.html ADDED
File without changes
data/tmp/x.txt ADDED
@@ -0,0 +1 @@
1
+
metadata ADDED
@@ -0,0 +1,122 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: outrider
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ platform: ruby
6
+ authors:
7
+ - Jaap Badlands
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2015-05-06 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: bundler
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ~>
18
+ - !ruby/object:Gem::Version
19
+ version: '1.9'
20
+ type: :development
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - ~>
25
+ - !ruby/object:Gem::Version
26
+ version: '1.9'
27
+ - !ruby/object:Gem::Dependency
28
+ name: rspec
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - '>='
32
+ - !ruby/object:Gem::Version
33
+ version: '0'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - '>='
39
+ - !ruby/object:Gem::Version
40
+ version: '0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: rake
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ~>
46
+ - !ruby/object:Gem::Version
47
+ version: '10.0'
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ~>
53
+ - !ruby/object:Gem::Version
54
+ version: '10.0'
55
+ description: 'Outrider''s purpose is to provide an easy-to-use programming interface
56
+ and organisational structure, to create and run tasks that can automatically visit,
57
+ interact with and test websites and also that process, clean and store data, and
58
+ tools for statistical analysis. '
59
+ email:
60
+ - jaap@deadlysyntax.com
61
+ executables:
62
+ - outrider
63
+ extensions: []
64
+ extra_rdoc_files: []
65
+ files:
66
+ - .gitignore
67
+ - .rspec
68
+ - .ruby-version
69
+ - .travis.yml
70
+ - Capfile
71
+ - Gemfile
72
+ - Gemfile.lock
73
+ - LICENSE.txt
74
+ - README.md
75
+ - Rakefile
76
+ - app/run.rb
77
+ - bin/console
78
+ - bin/outrider
79
+ - bin/setup
80
+ - config.ru
81
+ - config/messages.yml
82
+ - config/schema.sql
83
+ - lib/ignite.rb
84
+ - lib/outrider.rb
85
+ - lib/outrider/commandify.rb
86
+ - lib/outrider/engine.rb
87
+ - lib/outrider/intel.rb
88
+ - lib/outrider/project.rb
89
+ - lib/outrider/tools.rb
90
+ - lib/outrider/version.rb
91
+ - outrider.gemspec
92
+ - projects/nz_herald/auxiliary.rb
93
+ - projects/stuff/auxiliary.rb
94
+ - projects/test_project/auxiliary.rb
95
+ - projects/theage/auxiliary.rb
96
+ - public/index.html
97
+ - tmp/x.txt
98
+ homepage: https://github.com/deadlysyntax/outrider
99
+ licenses: []
100
+ metadata: {}
101
+ post_install_message:
102
+ rdoc_options: []
103
+ require_paths:
104
+ - lib
105
+ required_ruby_version: !ruby/object:Gem::Requirement
106
+ requirements:
107
+ - - '>='
108
+ - !ruby/object:Gem::Version
109
+ version: '0'
110
+ required_rubygems_version: !ruby/object:Gem::Requirement
111
+ requirements:
112
+ - - '>='
113
+ - !ruby/object:Gem::Version
114
+ version: '0'
115
+ requirements: []
116
+ rubyforge_project:
117
+ rubygems_version: 2.0.14
118
+ signing_key:
119
+ specification_version: 4
120
+ summary: Outrider Web Automation Framework provides structure and tools for writing
121
+ web-automation tasks
122
+ test_files: []