extraloop 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (27) hide show
  1. data/History.txt +2 -0
  2. data/README.md +135 -0
  3. data/examples/google_news_scraper.rb +22 -0
  4. data/examples/wikipedia_categories.rb +49 -0
  5. data/lib/extraloop/dom_extractor.rb +45 -0
  6. data/lib/extraloop/extraction_environment.rb +20 -0
  7. data/lib/extraloop/extraction_loop.rb +46 -0
  8. data/lib/extraloop/extractor_base.rb +40 -0
  9. data/lib/extraloop/hookable.rb +26 -0
  10. data/lib/extraloop/iterative_scraper.rb +291 -0
  11. data/lib/extraloop/json_extractor.rb +36 -0
  12. data/lib/extraloop/loggable.rb +64 -0
  13. data/lib/extraloop/scraper_base.rb +166 -0
  14. data/lib/extraloop/utils.rb +75 -0
  15. data/lib/extraloop.rb +43 -0
  16. data/spec/dom_extractor_spec.rb +165 -0
  17. data/spec/extraction_loop_spec.rb +76 -0
  18. data/spec/fixtures/doc.html +1324 -0
  19. data/spec/fixtures/doc.json +1 -0
  20. data/spec/helpers/scraper_helper.rb +46 -0
  21. data/spec/helpers/spec_helper.rb +12 -0
  22. data/spec/iterative_scraper_spec.rb +175 -0
  23. data/spec/json_extractor_spec.rb +146 -0
  24. data/spec/loggable_spec.rb +25 -0
  25. data/spec/scraper_base_spec.rb +178 -0
  26. data/spec/utils_spec.rb +44 -0
  27. metadata +140 -0
metadata ADDED
@@ -0,0 +1,140 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: extraloop
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ prerelease:
6
+ platform: ruby
7
+ authors:
8
+ - Andrea Fiore
9
+ autorequire:
10
+ bindir: bin
11
+ cert_chain: []
12
+ date: 2012-01-01 00:00:00.000000000Z
13
+ dependencies:
14
+ - !ruby/object:Gem::Dependency
15
+ name: yajl-ruby
16
+ requirement: &20994500 !ruby/object:Gem::Requirement
17
+ none: false
18
+ requirements:
19
+ - - ~>
20
+ - !ruby/object:Gem::Version
21
+ version: 1.1.0
22
+ type: :runtime
23
+ prerelease: false
24
+ version_requirements: *20994500
25
+ - !ruby/object:Gem::Dependency
26
+ name: nokogiri
27
+ requirement: &20945340 !ruby/object:Gem::Requirement
28
+ none: false
29
+ requirements:
30
+ - - ~>
31
+ - !ruby/object:Gem::Version
32
+ version: 1.5.0
33
+ type: :runtime
34
+ prerelease: false
35
+ version_requirements: *20945340
36
+ - !ruby/object:Gem::Dependency
37
+ name: typhoeus
38
+ requirement: &20944060 !ruby/object:Gem::Requirement
39
+ none: false
40
+ requirements:
41
+ - - ~>
42
+ - !ruby/object:Gem::Version
43
+ version: 0.3.2
44
+ type: :runtime
45
+ prerelease: false
46
+ version_requirements: *20944060
47
+ - !ruby/object:Gem::Dependency
48
+ name: logging
49
+ requirement: &20941960 !ruby/object:Gem::Requirement
50
+ none: false
51
+ requirements:
52
+ - - ~>
53
+ - !ruby/object:Gem::Version
54
+ version: 0.6.0
55
+ type: :runtime
56
+ prerelease: false
57
+ version_requirements: *20941960
58
+ - !ruby/object:Gem::Dependency
59
+ name: rspec
60
+ requirement: &20939500 !ruby/object:Gem::Requirement
61
+ none: false
62
+ requirements:
63
+ - - ~>
64
+ - !ruby/object:Gem::Version
65
+ version: 2.7.1
66
+ type: :development
67
+ prerelease: false
68
+ version_requirements: *20939500
69
+ - !ruby/object:Gem::Dependency
70
+ name: rr
71
+ requirement: &20887280 !ruby/object:Gem::Requirement
72
+ none: false
73
+ requirements:
74
+ - - ~>
75
+ - !ruby/object:Gem::Version
76
+ version: 1.0.4
77
+ type: :development
78
+ prerelease: false
79
+ version_requirements: *20887280
80
+ description: A Ruby library for extracting data from websites and web based APIs.
81
+ Supports most common document formats (i.e. HTML, XML, and JSON), and comes with
82
+ a handy mechanism for iterating over paginated datasets.
83
+ email: andrea.giulio.fiore@googlemail.com
84
+ executables: []
85
+ extensions: []
86
+ extra_rdoc_files: []
87
+ files:
88
+ - History.txt
89
+ - README.md
90
+ - examples/google_news_scraper.rb
91
+ - examples/wikipedia_categories.rb
92
+ - lib/extraloop.rb
93
+ - lib/extraloop/dom_extractor.rb
94
+ - lib/extraloop/extraction_environment.rb
95
+ - lib/extraloop/extraction_loop.rb
96
+ - lib/extraloop/extractor_base.rb
97
+ - lib/extraloop/hookable.rb
98
+ - lib/extraloop/iterative_scraper.rb
99
+ - lib/extraloop/json_extractor.rb
100
+ - lib/extraloop/loggable.rb
101
+ - lib/extraloop/scraper_base.rb
102
+ - lib/extraloop/utils.rb
103
+ - spec/dom_extractor_spec.rb
104
+ - spec/extraction_loop_spec.rb
105
+ - spec/fixtures/doc.html
106
+ - spec/fixtures/doc.json
107
+ - spec/helpers/scraper_helper.rb
108
+ - spec/helpers/spec_helper.rb
109
+ - spec/iterative_scraper_spec.rb
110
+ - spec/json_extractor_spec.rb
111
+ - spec/loggable_spec.rb
112
+ - spec/scraper_base_spec.rb
113
+ - spec/utils_spec.rb
114
+ homepage: http://github.com/afiore/extra_loop
115
+ licenses: []
116
+ post_install_message:
117
+ rdoc_options:
118
+ - --charset=UTF-8
119
+ require_paths:
120
+ - lib
121
+ required_ruby_version: !ruby/object:Gem::Requirement
122
+ none: false
123
+ requirements:
124
+ - - ! '>='
125
+ - !ruby/object:Gem::Version
126
+ version: '0'
127
+ required_rubygems_version: !ruby/object:Gem::Requirement
128
+ none: false
129
+ requirements:
130
+ - - ! '>='
131
+ - !ruby/object:Gem::Version
132
+ version: '0'
133
+ requirements: []
134
+ rubyforge_project: extraloop
135
+ rubygems_version: 1.8.10
136
+ signing_key:
137
+ specification_version: 3
138
+ summary: A toolkit for online data extraction.
139
+ test_files: []
140
+ has_rdoc: