saper 0.5.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (111) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +21 -0
  3. data/README.md +126 -0
  4. data/Rakefile +17 -0
  5. data/bin/saper +60 -0
  6. data/lib/lib/json_search.rb +54 -0
  7. data/lib/lib/mechanize.rb +26 -0
  8. data/lib/lib/nokogiri.rb +12 -0
  9. data/lib/saper.rb +37 -0
  10. data/lib/saper/actions/append_with.rb +14 -0
  11. data/lib/saper/actions/convert_to_html.rb +14 -0
  12. data/lib/saper/actions/convert_to_json.rb +14 -0
  13. data/lib/saper/actions/convert_to_markdown.rb +13 -0
  14. data/lib/saper/actions/convert_to_time.rb +15 -0
  15. data/lib/saper/actions/convert_to_xml.rb +14 -0
  16. data/lib/saper/actions/create_atom.rb +18 -0
  17. data/lib/saper/actions/fetch.rb +17 -0
  18. data/lib/saper/actions/find.rb +18 -0
  19. data/lib/saper/actions/find_first.rb +16 -0
  20. data/lib/saper/actions/get_attribute.rb +15 -0
  21. data/lib/saper/actions/get_contents.rb +14 -0
  22. data/lib/saper/actions/get_text.rb +14 -0
  23. data/lib/saper/actions/prepend_with.rb +14 -0
  24. data/lib/saper/actions/remove_after.rb +14 -0
  25. data/lib/saper/actions/remove_before.rb +14 -0
  26. data/lib/saper/actions/remove_matching.rb +14 -0
  27. data/lib/saper/actions/remove_tags.rb +15 -0
  28. data/lib/saper/actions/replace.rb +15 -0
  29. data/lib/saper/actions/run_recipe.rb +24 -0
  30. data/lib/saper/actions/run_recipe_and_save.rb +22 -0
  31. data/lib/saper/actions/save.rb +14 -0
  32. data/lib/saper/actions/select_matching.rb +14 -0
  33. data/lib/saper/actions/set_input.rb +19 -0
  34. data/lib/saper/actions/skip_tags.rb +15 -0
  35. data/lib/saper/actions/split.rb +24 -0
  36. data/lib/saper/arguments/attribute.rb +11 -0
  37. data/lib/saper/arguments/recipe.rb +42 -0
  38. data/lib/saper/arguments/text.rb +11 -0
  39. data/lib/saper/arguments/timezone.rb +11 -0
  40. data/lib/saper/arguments/variable.rb +11 -0
  41. data/lib/saper/arguments/xpath.rb +11 -0
  42. data/lib/saper/core/action.rb +209 -0
  43. data/lib/saper/core/argument.rb +106 -0
  44. data/lib/saper/core/browser.rb +87 -0
  45. data/lib/saper/core/dsl.rb +68 -0
  46. data/lib/saper/core/error.rb +47 -0
  47. data/lib/saper/core/item.rb +70 -0
  48. data/lib/saper/core/keychain.rb +18 -0
  49. data/lib/saper/core/logger.rb +74 -0
  50. data/lib/saper/core/namespace.rb +139 -0
  51. data/lib/saper/core/recipe.rb +134 -0
  52. data/lib/saper/core/runtime.rb +237 -0
  53. data/lib/saper/core/type.rb +45 -0
  54. data/lib/saper/items/atom.rb +64 -0
  55. data/lib/saper/items/document.rb +66 -0
  56. data/lib/saper/items/html.rb +85 -0
  57. data/lib/saper/items/json.rb +67 -0
  58. data/lib/saper/items/markdown.rb +36 -0
  59. data/lib/saper/items/nothing.rb +15 -0
  60. data/lib/saper/items/text.rb +54 -0
  61. data/lib/saper/items/time.rb +42 -0
  62. data/lib/saper/items/url.rb +34 -0
  63. data/lib/saper/items/xml.rb +79 -0
  64. data/lib/saper/version.rb +3 -0
  65. data/spec/actions/append_with_spec.rb +30 -0
  66. data/spec/actions/convert_to_html_spec.rb +24 -0
  67. data/spec/actions/convert_to_json_spec.rb +24 -0
  68. data/spec/actions/convert_to_markdown_spec.rb +24 -0
  69. data/spec/actions/convert_to_time_spec.rb +37 -0
  70. data/spec/actions/convert_to_xml_spec.rb +24 -0
  71. data/spec/actions/create_atom_spec.rb +31 -0
  72. data/spec/actions/fetch_spec.rb +7 -0
  73. data/spec/actions/find_first_spec.rb +7 -0
  74. data/spec/actions/find_spec.rb +7 -0
  75. data/spec/actions/get_attribute_spec.rb +7 -0
  76. data/spec/actions/get_contents.rb +7 -0
  77. data/spec/actions/get_text.rb +7 -0
  78. data/spec/actions/prepend_with_spec.rb +30 -0
  79. data/spec/actions/remove_after.rb +7 -0
  80. data/spec/actions/remove_before.rb +7 -0
  81. data/spec/actions/replace_spec.rb +7 -0
  82. data/spec/actions/run_recipe_and_save_spec.tmp.rb +52 -0
  83. data/spec/actions/run_recipe_spec.tmp.rb +53 -0
  84. data/spec/actions/save_spec.rb +7 -0
  85. data/spec/actions/select_matching_spec.rb +7 -0
  86. data/spec/actions/set_input_spec.rb +7 -0
  87. data/spec/actions/skip_tags_spec.rb +7 -0
  88. data/spec/actions/split_spec.rb +7 -0
  89. data/spec/core/action_spec.rb +151 -0
  90. data/spec/core/argument_spec.rb +79 -0
  91. data/spec/core/browser_spec.rb +7 -0
  92. data/spec/core/dsl_spec.rb +7 -0
  93. data/spec/core/item_spec.rb +7 -0
  94. data/spec/core/keychain_spec.rb +7 -0
  95. data/spec/core/logger_spec.rb +7 -0
  96. data/spec/core/namespace_spec.rb +18 -0
  97. data/spec/core/recipe_spec.rb +81 -0
  98. data/spec/core/runtime_spec.rb +165 -0
  99. data/spec/core/type_spec.rb +7 -0
  100. data/spec/items/atom_spec.rb +7 -0
  101. data/spec/items/document_spec.rb +7 -0
  102. data/spec/items/html_spec.rb +7 -0
  103. data/spec/items/json_spec.rb +7 -0
  104. data/spec/items/markdown_spec.rb +7 -0
  105. data/spec/items/nothing_spec.rb +7 -0
  106. data/spec/items/text_spec.rb +17 -0
  107. data/spec/items/time_spec.rb +7 -0
  108. data/spec/items/url_spec.rb +7 -0
  109. data/spec/items/xml_spec.rb +17 -0
  110. data/spec/spec_helper.rb +22 -0
  111. metadata +355 -0
@@ -0,0 +1,17 @@
1
+ require './spec/spec_helper'
2
+
3
+ describe Saper::Items::Text do
4
+
5
+ context ".new(String)" do
6
+ it "returns Text instance" do
7
+ Saper::Items::Text.new("text").should be_a(Saper::Items::Text)
8
+ end
9
+ end
10
+
11
+ context ".new(nil)" do
12
+ it "raises InvalidItem" do
13
+ expect { Saper::Items::Text.new(nil) }.to raise_error(Saper::InvalidItem)
14
+ end
15
+ end
16
+
17
+ end
@@ -0,0 +1,7 @@
1
+ require './spec/spec_helper'
2
+
3
+ describe Saper::Items::Time do
4
+
5
+ it # TODO
6
+
7
+ end
@@ -0,0 +1,7 @@
1
+ require './spec/spec_helper'
2
+
3
+ describe Saper::Items::URL do
4
+
5
+ it # TODO
6
+
7
+ end
@@ -0,0 +1,17 @@
1
+ require './spec/spec_helper'
2
+
3
+ describe Saper::Items::XML do
4
+
5
+ context ".new(String)" do
6
+ it "returns Text instance" do
7
+ Saper::Items::Text.new("text").should be_a(Saper::Items::Text)
8
+ end
9
+ end
10
+
11
+ context ".new(nil)" do
12
+ it "raises InvalidItem" do
13
+ expect { Saper::Items::XML.new(nil) }.to raise_error(Saper::InvalidItem)
14
+ end
15
+ end
16
+
17
+ end
@@ -0,0 +1,22 @@
1
+ require 'saper'
2
+
3
+ class EmptyAction < Saper::Action
4
+ # default configuration
5
+ end
6
+
7
+ class TestAction < Saper::Action
8
+ accepts :text
9
+ end
10
+
11
+ class ActionWithMandatoryArgument < Saper::Action
12
+ accepts :text
13
+ argument :text
14
+ run { |input, string| "%s %s" % [string, input] }
15
+ end
16
+
17
+ class ActionWithOptionalArgument < Saper::Action
18
+ accepts :text
19
+ argument :text, :optional => true
20
+ run { |input, string| "%s %s" % [string, input] }
21
+ end
22
+
metadata ADDED
@@ -0,0 +1,355 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: saper
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.5.0
5
+ platform: ruby
6
+ authors:
7
+ - Alex Serebryakov
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2014-05-04 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: mechanize
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ~>
18
+ - !ruby/object:Gem::Version
19
+ version: 2.7.3
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - ~>
25
+ - !ruby/object:Gem::Version
26
+ version: 2.7.3
27
+ - !ruby/object:Gem::Dependency
28
+ name: nokogiri
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ~>
32
+ - !ruby/object:Gem::Version
33
+ version: 1.6.1
34
+ type: :runtime
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ~>
39
+ - !ruby/object:Gem::Version
40
+ version: 1.6.1
41
+ - !ruby/object:Gem::Dependency
42
+ name: json
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ~>
46
+ - !ruby/object:Gem::Version
47
+ version: 1.8.1
48
+ type: :runtime
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ~>
53
+ - !ruby/object:Gem::Version
54
+ version: 1.8.1
55
+ - !ruby/object:Gem::Dependency
56
+ name: vremya
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - ~>
60
+ - !ruby/object:Gem::Version
61
+ version: 1.0.0
62
+ type: :runtime
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - ~>
67
+ - !ruby/object:Gem::Version
68
+ version: 1.0.0
69
+ - !ruby/object:Gem::Dependency
70
+ name: reverse_markdown
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - ~>
74
+ - !ruby/object:Gem::Version
75
+ version: 0.5.1
76
+ type: :runtime
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - ~>
81
+ - !ruby/object:Gem::Version
82
+ version: 0.5.1
83
+ - !ruby/object:Gem::Dependency
84
+ name: trollop
85
+ requirement: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - ~>
88
+ - !ruby/object:Gem::Version
89
+ version: 2.0.0
90
+ type: :runtime
91
+ prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - ~>
95
+ - !ruby/object:Gem::Version
96
+ version: 2.0.0
97
+ - !ruby/object:Gem::Dependency
98
+ name: rake
99
+ requirement: !ruby/object:Gem::Requirement
100
+ requirements:
101
+ - - ~>
102
+ - !ruby/object:Gem::Version
103
+ version: 10.3.1
104
+ type: :development
105
+ prerelease: false
106
+ version_requirements: !ruby/object:Gem::Requirement
107
+ requirements:
108
+ - - ~>
109
+ - !ruby/object:Gem::Version
110
+ version: 10.3.1
111
+ - !ruby/object:Gem::Dependency
112
+ name: rspec
113
+ requirement: !ruby/object:Gem::Requirement
114
+ requirements:
115
+ - - ~>
116
+ - !ruby/object:Gem::Version
117
+ version: 2.14.1
118
+ type: :development
119
+ prerelease: false
120
+ version_requirements: !ruby/object:Gem::Requirement
121
+ requirements:
122
+ - - ~>
123
+ - !ruby/object:Gem::Version
124
+ version: 2.14.1
125
+ - !ruby/object:Gem::Dependency
126
+ name: yard
127
+ requirement: !ruby/object:Gem::Requirement
128
+ requirements:
129
+ - - ~>
130
+ - !ruby/object:Gem::Version
131
+ version: 0.8.7
132
+ type: :development
133
+ prerelease: false
134
+ version_requirements: !ruby/object:Gem::Requirement
135
+ requirements:
136
+ - - ~>
137
+ - !ruby/object:Gem::Version
138
+ version: 0.8.7
139
+ - !ruby/object:Gem::Dependency
140
+ name: guard
141
+ requirement: !ruby/object:Gem::Requirement
142
+ requirements:
143
+ - - ~>
144
+ - !ruby/object:Gem::Version
145
+ version: 2.6.0
146
+ type: :development
147
+ prerelease: false
148
+ version_requirements: !ruby/object:Gem::Requirement
149
+ requirements:
150
+ - - ~>
151
+ - !ruby/object:Gem::Version
152
+ version: 2.6.0
153
+ - !ruby/object:Gem::Dependency
154
+ name: guard-rspec
155
+ requirement: !ruby/object:Gem::Requirement
156
+ requirements:
157
+ - - ~>
158
+ - !ruby/object:Gem::Version
159
+ version: 4.2.8
160
+ type: :development
161
+ prerelease: false
162
+ version_requirements: !ruby/object:Gem::Requirement
163
+ requirements:
164
+ - - ~>
165
+ - !ruby/object:Gem::Version
166
+ version: 4.2.8
167
+ description: Web automation library that allows to crawl websites and extract data
168
+ in an efficient and fault-tolerant manner.
169
+ email: alex@merimond.com
170
+ executables:
171
+ - saper
172
+ extensions: []
173
+ extra_rdoc_files: []
174
+ files:
175
+ - bin/saper
176
+ - lib/lib/json_search.rb
177
+ - lib/lib/mechanize.rb
178
+ - lib/lib/nokogiri.rb
179
+ - lib/saper/actions/append_with.rb
180
+ - lib/saper/actions/convert_to_html.rb
181
+ - lib/saper/actions/convert_to_json.rb
182
+ - lib/saper/actions/convert_to_markdown.rb
183
+ - lib/saper/actions/convert_to_time.rb
184
+ - lib/saper/actions/convert_to_xml.rb
185
+ - lib/saper/actions/create_atom.rb
186
+ - lib/saper/actions/fetch.rb
187
+ - lib/saper/actions/find.rb
188
+ - lib/saper/actions/find_first.rb
189
+ - lib/saper/actions/get_attribute.rb
190
+ - lib/saper/actions/get_contents.rb
191
+ - lib/saper/actions/get_text.rb
192
+ - lib/saper/actions/prepend_with.rb
193
+ - lib/saper/actions/remove_after.rb
194
+ - lib/saper/actions/remove_before.rb
195
+ - lib/saper/actions/remove_matching.rb
196
+ - lib/saper/actions/remove_tags.rb
197
+ - lib/saper/actions/replace.rb
198
+ - lib/saper/actions/run_recipe.rb
199
+ - lib/saper/actions/run_recipe_and_save.rb
200
+ - lib/saper/actions/save.rb
201
+ - lib/saper/actions/select_matching.rb
202
+ - lib/saper/actions/set_input.rb
203
+ - lib/saper/actions/skip_tags.rb
204
+ - lib/saper/actions/split.rb
205
+ - lib/saper/arguments/attribute.rb
206
+ - lib/saper/arguments/recipe.rb
207
+ - lib/saper/arguments/text.rb
208
+ - lib/saper/arguments/timezone.rb
209
+ - lib/saper/arguments/variable.rb
210
+ - lib/saper/arguments/xpath.rb
211
+ - lib/saper/core/action.rb
212
+ - lib/saper/core/argument.rb
213
+ - lib/saper/core/browser.rb
214
+ - lib/saper/core/dsl.rb
215
+ - lib/saper/core/error.rb
216
+ - lib/saper/core/item.rb
217
+ - lib/saper/core/keychain.rb
218
+ - lib/saper/core/logger.rb
219
+ - lib/saper/core/namespace.rb
220
+ - lib/saper/core/recipe.rb
221
+ - lib/saper/core/runtime.rb
222
+ - lib/saper/core/type.rb
223
+ - lib/saper/items/atom.rb
224
+ - lib/saper/items/document.rb
225
+ - lib/saper/items/html.rb
226
+ - lib/saper/items/json.rb
227
+ - lib/saper/items/markdown.rb
228
+ - lib/saper/items/nothing.rb
229
+ - lib/saper/items/text.rb
230
+ - lib/saper/items/time.rb
231
+ - lib/saper/items/url.rb
232
+ - lib/saper/items/xml.rb
233
+ - lib/saper/version.rb
234
+ - lib/saper.rb
235
+ - Rakefile
236
+ - README.md
237
+ - LICENSE
238
+ - spec/actions/append_with_spec.rb
239
+ - spec/actions/convert_to_html_spec.rb
240
+ - spec/actions/convert_to_json_spec.rb
241
+ - spec/actions/convert_to_markdown_spec.rb
242
+ - spec/actions/convert_to_time_spec.rb
243
+ - spec/actions/convert_to_xml_spec.rb
244
+ - spec/actions/create_atom_spec.rb
245
+ - spec/actions/fetch_spec.rb
246
+ - spec/actions/find_first_spec.rb
247
+ - spec/actions/find_spec.rb
248
+ - spec/actions/get_attribute_spec.rb
249
+ - spec/actions/get_contents.rb
250
+ - spec/actions/get_text.rb
251
+ - spec/actions/prepend_with_spec.rb
252
+ - spec/actions/remove_after.rb
253
+ - spec/actions/remove_before.rb
254
+ - spec/actions/replace_spec.rb
255
+ - spec/actions/run_recipe_and_save_spec.tmp.rb
256
+ - spec/actions/run_recipe_spec.tmp.rb
257
+ - spec/actions/save_spec.rb
258
+ - spec/actions/select_matching_spec.rb
259
+ - spec/actions/set_input_spec.rb
260
+ - spec/actions/skip_tags_spec.rb
261
+ - spec/actions/split_spec.rb
262
+ - spec/core/action_spec.rb
263
+ - spec/core/argument_spec.rb
264
+ - spec/core/browser_spec.rb
265
+ - spec/core/dsl_spec.rb
266
+ - spec/core/item_spec.rb
267
+ - spec/core/keychain_spec.rb
268
+ - spec/core/logger_spec.rb
269
+ - spec/core/namespace_spec.rb
270
+ - spec/core/recipe_spec.rb
271
+ - spec/core/runtime_spec.rb
272
+ - spec/core/type_spec.rb
273
+ - spec/items/atom_spec.rb
274
+ - spec/items/document_spec.rb
275
+ - spec/items/html_spec.rb
276
+ - spec/items/json_spec.rb
277
+ - spec/items/markdown_spec.rb
278
+ - spec/items/nothing_spec.rb
279
+ - spec/items/text_spec.rb
280
+ - spec/items/time_spec.rb
281
+ - spec/items/url_spec.rb
282
+ - spec/items/xml_spec.rb
283
+ - spec/spec_helper.rb
284
+ homepage: https://github.com/merimond/saper
285
+ licenses:
286
+ - MIT
287
+ metadata: {}
288
+ post_install_message:
289
+ rdoc_options: []
290
+ require_paths:
291
+ - lib
292
+ required_ruby_version: !ruby/object:Gem::Requirement
293
+ requirements:
294
+ - - '>='
295
+ - !ruby/object:Gem::Version
296
+ version: '0'
297
+ required_rubygems_version: !ruby/object:Gem::Requirement
298
+ requirements:
299
+ - - '>='
300
+ - !ruby/object:Gem::Version
301
+ version: '0'
302
+ requirements: []
303
+ rubyforge_project:
304
+ rubygems_version: 2.0.3
305
+ signing_key:
306
+ specification_version: 4
307
+ summary: Ruby data scraping library
308
+ test_files:
309
+ - spec/actions/append_with_spec.rb
310
+ - spec/actions/convert_to_html_spec.rb
311
+ - spec/actions/convert_to_json_spec.rb
312
+ - spec/actions/convert_to_markdown_spec.rb
313
+ - spec/actions/convert_to_time_spec.rb
314
+ - spec/actions/convert_to_xml_spec.rb
315
+ - spec/actions/create_atom_spec.rb
316
+ - spec/actions/fetch_spec.rb
317
+ - spec/actions/find_first_spec.rb
318
+ - spec/actions/find_spec.rb
319
+ - spec/actions/get_attribute_spec.rb
320
+ - spec/actions/get_contents.rb
321
+ - spec/actions/get_text.rb
322
+ - spec/actions/prepend_with_spec.rb
323
+ - spec/actions/remove_after.rb
324
+ - spec/actions/remove_before.rb
325
+ - spec/actions/replace_spec.rb
326
+ - spec/actions/run_recipe_and_save_spec.tmp.rb
327
+ - spec/actions/run_recipe_spec.tmp.rb
328
+ - spec/actions/save_spec.rb
329
+ - spec/actions/select_matching_spec.rb
330
+ - spec/actions/set_input_spec.rb
331
+ - spec/actions/skip_tags_spec.rb
332
+ - spec/actions/split_spec.rb
333
+ - spec/core/action_spec.rb
334
+ - spec/core/argument_spec.rb
335
+ - spec/core/browser_spec.rb
336
+ - spec/core/dsl_spec.rb
337
+ - spec/core/item_spec.rb
338
+ - spec/core/keychain_spec.rb
339
+ - spec/core/logger_spec.rb
340
+ - spec/core/namespace_spec.rb
341
+ - spec/core/recipe_spec.rb
342
+ - spec/core/runtime_spec.rb
343
+ - spec/core/type_spec.rb
344
+ - spec/items/atom_spec.rb
345
+ - spec/items/document_spec.rb
346
+ - spec/items/html_spec.rb
347
+ - spec/items/json_spec.rb
348
+ - spec/items/markdown_spec.rb
349
+ - spec/items/nothing_spec.rb
350
+ - spec/items/text_spec.rb
351
+ - spec/items/time_spec.rb
352
+ - spec/items/url_spec.rb
353
+ - spec/items/xml_spec.rb
354
+ - spec/spec_helper.rb
355
+ has_rdoc: