saper 0.5.1 → 0.5.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (120) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +3 -0
  3. data/Rakefile +6 -6
  4. data/bin/saper +17 -36
  5. data/lib/saper.rb +63 -20
  6. data/lib/saper/actions/append_with.rb +1 -1
  7. data/lib/saper/actions/convert_to_html.rb +1 -1
  8. data/lib/saper/actions/convert_to_json.rb +6 -2
  9. data/lib/saper/actions/convert_to_markdown.rb +1 -1
  10. data/lib/saper/actions/convert_to_time.rb +38 -1
  11. data/lib/saper/actions/convert_to_xml.rb +1 -1
  12. data/lib/saper/actions/create_atom.rb +6 -2
  13. data/lib/saper/actions/fetch.rb +3 -2
  14. data/lib/saper/actions/fetch_with_token.rb +18 -0
  15. data/lib/saper/actions/find.rb +1 -1
  16. data/lib/saper/actions/find_first.rb +1 -1
  17. data/lib/saper/actions/get_attribute.rb +7 -2
  18. data/lib/saper/actions/get_contents.rb +1 -1
  19. data/lib/saper/actions/get_text.rb +1 -1
  20. data/lib/saper/actions/nothing.rb +11 -0
  21. data/lib/saper/actions/prepend_with.rb +2 -2
  22. data/lib/saper/actions/remove_after.rb +2 -2
  23. data/lib/saper/actions/remove_before.rb +2 -2
  24. data/lib/saper/actions/remove_matching.rb +2 -2
  25. data/lib/saper/actions/remove_tags.rb +1 -1
  26. data/lib/saper/actions/replace.rb +1 -1
  27. data/lib/saper/actions/run_recipe.rb +2 -2
  28. data/lib/saper/actions/run_recipe_and_save.rb +3 -3
  29. data/lib/saper/actions/save.rb +2 -2
  30. data/lib/saper/actions/select_matching.rb +2 -2
  31. data/lib/saper/actions/set_input.rb +1 -1
  32. data/lib/saper/actions/skip_tags.rb +1 -1
  33. data/lib/saper/actions/split.rb +1 -1
  34. data/lib/saper/arguments/attribute.rb +6 -0
  35. data/lib/saper/arguments/recipe.rb +12 -6
  36. data/lib/saper/arguments/service.rb +12 -0
  37. data/lib/saper/arguments/text.rb +1 -0
  38. data/lib/saper/arguments/timezone.rb +2 -1
  39. data/lib/saper/arguments/url.rb +12 -0
  40. data/lib/saper/arguments/variable.rb +2 -1
  41. data/lib/saper/arguments/xpath.rb +2 -1
  42. data/lib/saper/core/action.rb +43 -13
  43. data/lib/saper/core/argument.rb +30 -14
  44. data/lib/saper/core/browser.rb +29 -28
  45. data/lib/saper/core/dsl.rb +14 -16
  46. data/lib/saper/core/error.rb +1 -1
  47. data/lib/saper/core/item.rb +2 -2
  48. data/lib/saper/core/keychain.rb +58 -5
  49. data/lib/saper/core/namespace.rb +15 -18
  50. data/lib/saper/core/recipe.rb +53 -12
  51. data/lib/saper/core/result.rb +72 -0
  52. data/lib/saper/core/runtime.rb +44 -177
  53. data/lib/saper/core/stack.rb +57 -0
  54. data/lib/saper/items/atom.rb +18 -1
  55. data/lib/saper/items/document.rb +39 -19
  56. data/lib/saper/items/html.rb +52 -7
  57. data/lib/saper/items/json.rb +19 -1
  58. data/lib/saper/items/markdown.rb +14 -4
  59. data/lib/saper/items/nothing.rb +6 -0
  60. data/lib/saper/items/text.rb +28 -3
  61. data/lib/saper/items/time.rb +25 -2
  62. data/lib/saper/items/url.rb +8 -2
  63. data/lib/saper/items/xml.rb +51 -11
  64. data/lib/{lib → saper/patches}/mechanize.rb +0 -0
  65. data/lib/{lib → saper/patches}/nokogiri.rb +0 -0
  66. data/lib/saper/version.rb +2 -2
  67. data/spec/actions/append_with_spec.rb +29 -52
  68. data/spec/actions/convert_to_html_spec.rb +13 -30
  69. data/spec/actions/convert_to_json_spec.rb +13 -30
  70. data/spec/actions/convert_to_markdown_spec.rb +17 -19
  71. data/spec/actions/convert_to_time_spec.rb +25 -43
  72. data/spec/actions/convert_to_xml_spec.rb +15 -11
  73. data/spec/actions/create_atom_spec.rb +11 -19
  74. data/spec/actions/fetch_spec.rb +3 -8
  75. data/spec/actions/find_first_spec.rb +38 -40
  76. data/spec/actions/find_spec.rb +23 -39
  77. data/spec/actions/get_attribute_spec.rb +30 -3
  78. data/spec/actions/{get_contents.rb → get_contents_spec.rb} +0 -0
  79. data/spec/actions/{get_text.rb → get_text_spec.rb} +0 -0
  80. data/spec/actions/nothing_spec.rb +7 -0
  81. data/spec/actions/prepend_with_spec.rb +31 -18
  82. data/spec/actions/{remove_after.rb → remove_after_spec.rb} +0 -0
  83. data/spec/actions/{remove_before.rb → remove_before_spec.rb} +0 -0
  84. data/spec/actions/remove_matching_spec.rb +7 -0
  85. data/spec/actions/remove_tags_spec.rb +7 -0
  86. data/spec/actions/run_recipe_and_save_spec.rb +7 -0
  87. data/spec/actions/run_recipe_spec.rb +7 -0
  88. data/spec/arguments/attribute_spec.rb +7 -0
  89. data/spec/arguments/recipe_spec.rb +7 -0
  90. data/spec/arguments/text_spec.rb +7 -0
  91. data/spec/arguments/timezone_spec.rb +7 -0
  92. data/spec/arguments/variable_spec.rb +7 -0
  93. data/spec/arguments/xpath_spec.rb +7 -0
  94. data/spec/core/action_spec.rb +13 -142
  95. data/spec/core/argument_spec.rb +38 -71
  96. data/spec/core/browser_spec.rb +18 -3
  97. data/spec/core/dsl_spec.rb +1 -1
  98. data/spec/core/item_spec.rb +1 -1
  99. data/spec/core/namespace_spec.rb +0 -11
  100. data/spec/core/recipe_spec.rb +54 -77
  101. data/spec/core/result_spec.rb +7 -0
  102. data/spec/core/runtime_spec.rb +20 -151
  103. data/spec/core/stack_spec.rb +7 -0
  104. data/spec/integration/simple_invalid_spec.rb +39 -0
  105. data/spec/integration/simple_valid_spec.rb +39 -0
  106. data/spec/items/atom_spec.rb +6 -1
  107. data/spec/items/document_spec.rb +93 -15
  108. data/spec/items/html_spec.rb +45 -28
  109. data/spec/items/json_spec.rb +10 -10
  110. data/spec/items/markdown_spec.rb +24 -3
  111. data/spec/items/nothing_spec.rb +1 -1
  112. data/spec/items/text_spec.rb +13 -41
  113. data/spec/items/time_spec.rb +25 -4
  114. data/spec/items/url_spec.rb +1 -7
  115. data/spec/items/xml_spec.rb +46 -39
  116. data/spec/spec_helper.rb +2 -21
  117. metadata +63 -60
  118. data/lib/lib/json_search.rb +0 -54
  119. data/spec/actions/run_recipe_and_save_spec.tmp.rb +0 -52
  120. data/spec/actions/run_recipe_spec.tmp.rb +0 -53
@@ -1,52 +1,51 @@
1
1
  module Saper
2
+ # Runtime provides a set of auxiliary functions to Saper actions:
3
+ # browser requests, keychain access, storage of variables.
2
4
  class Runtime
3
5
 
4
- # Returns action input.
5
- attr_reader :input
6
-
7
- # Returns action output.
8
- attr_reader :output
9
-
10
- # Returns error instance (if any)
11
- attr_reader :error
12
-
13
- # Returns action instance.
14
- attr_reader :action
15
-
16
- # Returns embedded recipes
17
- attr_reader :subrecipes
6
+ # Returns Saper::Browser instance used by runtime.
7
+ # @return [Saper::Browser]
8
+ attr_reader :browser
18
9
 
19
- # Returns subsequent runtime instances.
20
- attr_reader :children
10
+ # Returns Saper::Keychain instance used by runtime.
11
+ # @return [Saper::Keychain]
12
+ attr_reader :keychain
21
13
 
22
- # Returns the size of action chain.
23
- attr_reader :depth
14
+ # Returns a Hash of variables stored by runtime.
15
+ # @return [Hash]
16
+ attr_reader :variables
24
17
 
25
18
  # Returns a new Saper::Runtime instance.
26
- # @param stack [Array<Saper::Action>] chain of actions
27
- # @param input [Object, nil] action input
19
+ # @param variables [Hash]
28
20
  # @param options [Hash]
21
+ # @option options [Browser] :browser Browser instance
22
+ # @option options [String, Keychain] :keychain Keychain file or instance
23
+ # @option options [String, Symbol] :agent User-Agent type or string
29
24
  # @return [Saper::Runtime]
30
- def initialize(stack = [], input = nil, options = {})
31
- if stack.is_a?(Recipe)
32
- stack = stack.actions
25
+ def initialize(variables = {}, options = {})
26
+ if options.nil?
27
+ puts self.inspect
28
+ raise 'stop'
29
+ end
30
+ if options[:keychain].is_a?(Keychain)
31
+ @keychain = options[:keychain]
32
+ end
33
+ if options[:keychain].is_a?(String)
34
+ @keychain = Keychain.load(options[:keychain])
35
+ end
36
+ unless @keychain.is_a?(Keychain)
37
+ @keychain = Keychain.new
33
38
  end
34
- @input = input
35
- @output = nil
36
- @error = nil
37
- @action = nil
38
- @children = []
39
- @subrecipes = []
40
- @options = options
41
- @depth = stack.size
42
- @options[:keychain] ||= Keychain.new
43
- @options[:logger] ||= Logger.new(@options[:log])
44
- @options[:browser] ||= Browser.new(:logger => logger)
45
- @options[:variables] ||= {}
39
+ if options[:browser].is_a?(Browser)
40
+ @browser = options[:browser]
41
+ end
42
+ unless @browser.is_a?(Browser)
43
+ @browser = Browser.new(:agent => options[:agent])
44
+ end
45
+ @variables = variables
46
46
  if block_given?
47
47
  yield self
48
48
  end
49
- execute(stack.dup)
50
49
  end
51
50
 
52
51
  # Returns volume of incoming traffic (in bytes)
@@ -71,7 +70,7 @@ module Saper
71
70
  # @param name [Symbol]
72
71
  # @return [Object]
73
72
  def [](name)
74
- @options[:variables][name]
73
+ @variables[name]
75
74
  end
76
75
 
77
76
  # Set variable value
@@ -79,62 +78,7 @@ module Saper
79
78
  # @param value [Object]
80
79
  # @return [Object]
81
80
  def []=(name, value)
82
- @options[:variables][name] = value
83
- end
84
-
85
- # Return a hash with all variables
86
- # @return [Hash]
87
- def variables
88
- @options[:variables].dup
89
- end
90
-
91
- # Returns descendant runtime instances at a given depth level.
92
- # @param level [Integer]
93
- # @return [Array<Saper::Runtime>]
94
- def descendants(level = 0)
95
- level < 1 ? [self] : children.map { |i| i.descendants(level - 1) }.flatten
96
- end
97
-
98
- # Returns runtime results.
99
- # @return [Array<Object>, Object]
100
- def results
101
- native_results
102
- end
103
-
104
- # Returns an array of results.
105
- # @return [Array]
106
- def result_array
107
- native_result_array
108
- end
109
-
110
- # Returns `true` if this or any of the subsequent actions produces multiple results.
111
- # @return [Boolean]
112
- def multiple?
113
- (action.nil? ? false : action.multiple?) || children.any?(&:multiple?)
114
- end
115
-
116
- # Returns Saper::Browser instance used by runtime.
117
- # @return [Saper::Browser]
118
- def browser
119
- @options[:browser]
120
- end
121
-
122
- # Returns Saper::Keychain instance used by runtime.
123
- # @return [Saper::Keychain]
124
- def keychain
125
- @options[:keychain]
126
- end
127
-
128
- # Returns Saper::Logger.
129
- # @return [Saper::Logger]
130
- def logger
131
- @options[:logger]
132
- end
133
-
134
- # Writes full runtime log to logger instance.
135
- # @return [void]
136
- def backtrace
137
- logger.runtime(self)
81
+ @variables[name] = value
138
82
  end
139
83
 
140
84
  # Returns access credentials for specified service
@@ -144,93 +88,16 @@ module Saper
144
88
  keychain[service]
145
89
  end
146
90
 
147
- # Runs an arbitrary recipe, specified by ID, block or as instance.
148
- # @param recipe [Saper::Recipe]
149
- # @param input [Object, nil]
91
+ # Returns a duplicate instance of self.
150
92
  # @return [Saper::Runtime]
151
- def recipe(recipe, input = nil, &block)
152
- unless recipe.is_a?(Recipe)
153
- raise RecipeNotFound, recipe
154
- end
155
- runtime = Runtime.new(recipe, input, @options.merge(:variables => {}))
156
- if runtime.error?
157
- @error = runtime.error
158
- end
159
- subrecipes << runtime
160
- runtime.non_native_results
161
- end
162
-
163
- # Returns `true` if action raised no errors.
164
- # @return [Boolean]
165
- def success?
166
- error.nil?
167
- end
168
-
169
- # Returns `true` if action raised at least one error.
170
- # @return [Boolean]
171
- def error?
172
- !success?
173
- end
174
-
175
- # Returns runtime results as JSON.
176
- # @return [Array]
177
- def to_json(*args)
178
- JSON.generate non_native_results.map(&:serialize)
179
- end
180
-
181
- # Returns runtime results as Saper::Item instances.
182
- # @return [Saper::Item, Array<Saper::Item>]
183
- def non_native_results
184
- multiple? ? non_native_result_array : non_native_result_array.first
185
- end
186
-
187
- # Returns runtime results as native Ruby objects.
188
- # @return [Object, Array<Object>]
189
- def native_results
190
- multiple? ? native_result_array : native_result_array.first
191
- end
192
-
193
- private
194
-
195
- # Returns runtime results as an array of Saper::Item instances.
196
- # @return [Array<Saper::Item>]
197
- def non_native_result_array
198
- descendants(depth - 1).map(&:output).flatten.compact
199
- end
200
-
201
- # Returns runtime results as an array of native Ruby objects.
202
- # @return [Array<Object>]
203
- def native_result_array
204
- non_native_result_array.map { |result| result.is_a?(Item) ? result.to_native : result }
93
+ def copy
94
+ Runtime.new(variables.dup, :browser => browser, :keychain => keychain)
205
95
  end
206
96
 
207
- # Executes recipes.
208
- # @return [self]
209
- def execute(stack)
210
- @output = input
211
- @action = stack.shift
212
- if action.nil?
213
- return self
214
- end
215
- unless action.is_a?(Saper::Action)
216
- raise ActionExpected, action
217
- end
218
- begin
219
- @output = action.run(input, self)
220
- rescue Saper::Error => e
221
- @output = nil
222
- @error = e
223
- end
224
- unless @output.is_a?(Array)
225
- @output = [@output]
226
- end
227
- if stack.empty?
228
- return self
229
- end
230
- @children = @output.map do |item|
231
- Runtime.new(stack, item, @options.dup)
232
- end
233
- self
97
+ # Returns a duplicate instance of self without the stored variables.
98
+ # @return [Saper::Runtime]
99
+ def copy_without_variables
100
+ Runtime.new({}, :browser => browser, :keychain => keychain)
234
101
  end
235
102
 
236
103
  end
@@ -0,0 +1,57 @@
1
+ module Saper
2
+ class Stack
3
+
4
+ # Returns Runtime instance used by the stack
5
+ # @return [Saper::Runtime]
6
+ attr_reader :runtime
7
+
8
+ # Returns all items in the stack.
9
+ # @return [Array<Saper::Item>]
10
+ attr_reader :items
11
+
12
+ # Returns a new Stack instance.
13
+ # @param runtime [Saper::Runtime]
14
+ # @param items [Saper::Item]
15
+ # @return [Saper::Stack]
16
+ def initialize(runtime, *items)
17
+ @runtime, @items = runtime, items.flatten
18
+ end
19
+
20
+ # Returns a duplicate instance of self.
21
+ # @return [Saper::Stack]
22
+ def copy
23
+ Stack.new(runtime.copy, *to_a)
24
+ end
25
+
26
+ # Adds action result to the end of the stack and returns self.
27
+ # @param item [Saper::Item]
28
+ # @return [self]
29
+ def add(item)
30
+ @items.push(item); self
31
+ end
32
+
33
+ # Returns the result of the last action or nil (in case of an error).
34
+ # @return [Saper::Item]
35
+ def result
36
+ error? ? Items::Nothing.new : last
37
+ end
38
+
39
+ # Returns the result of the last action.
40
+ # @return [Saper::Item]
41
+ def last
42
+ @items.last || Items::Nothing.new
43
+ end
44
+
45
+ # @todo
46
+ def error?
47
+ last.is_a?(Saper::Error)
48
+ end
49
+
50
+ # Returns an array of action results.
51
+ # @return [Array<Saper::Item>]
52
+ def to_a
53
+ @items.dup
54
+ end
55
+
56
+ end
57
+ end
@@ -2,6 +2,7 @@ module Saper
2
2
  module Items
3
3
  class Atom < Item
4
4
 
5
+ # @todo
5
6
  def self.new(item)
6
7
  super case item
7
8
  when Hash
@@ -11,22 +12,32 @@ module Saper
11
12
  end
12
13
  end
13
14
 
15
+ # @todo
14
16
  def initialize(hash)
15
17
  @atts = hash
16
18
  end
17
19
 
20
+ # @todo
21
+ def ==(other)
22
+ to_hash == other
23
+ end
24
+
25
+ # @todo
18
26
  def to_hash
19
27
  @atts.dup
20
28
  end
21
29
 
30
+ # @todo
22
31
  def [](name)
23
32
  @atts[name]
24
33
  end
25
34
 
35
+ # @todo
26
36
  def serialize
27
- @atts.dup
37
+ Hash[@atts.map { |key, value| [key, value.respond_to?(:serialize) ? value.serialize : value] }]
28
38
  end
29
39
 
40
+ # @todo
30
41
  def to_native(object = nil)
31
42
  if object.nil?
32
43
  return to_native(to_hash)
@@ -43,10 +54,16 @@ module Saper
43
54
  object
44
55
  end
45
56
 
57
+ # @todo
46
58
  def to_json
47
59
  JSON.new(@hash, true)
48
60
  end
49
61
 
62
+ # @todo
63
+ def to_s
64
+ to_json
65
+ end
66
+
50
67
  end
51
68
  end
52
69
  end
@@ -2,67 +2,87 @@ module Saper
2
2
  module Items
3
3
  class Document < Item
4
4
 
5
- def self.new(item)
6
- super case item
7
- when Mechanize::File
8
- item
9
- else
10
- raise(InvalidItem, item)
11
- end
12
- end
5
+ # @todo
6
+ attr_reader :body
13
7
 
14
- def initialize(mechanize)
15
- @mech = mechanize
16
- end
8
+ # @todo
9
+ attr_reader :uri
17
10
 
18
- def uri
19
- @mech.uri.to_s
11
+ # @todo
12
+ attr_reader :headers
13
+
14
+ # @todo
15
+ def self.new(body, uri = nil, headers = {})
16
+ unless body.is_a?(String)
17
+ raise(InvalidItem)
18
+ end
19
+ unless headers.is_a?(Hash)
20
+ raise(InvalidItem)
21
+ end
22
+ super
20
23
  end
21
24
 
22
- def body
23
- @mech.body
25
+ # @todo
26
+ def initialize(body, uri = nil, headers = {})
27
+ @uri = (uri.nil? ? nil : uri.to_s)
28
+ @body = body
29
+ @headers = headers
24
30
  end
25
31
 
32
+ # @todo
26
33
  def size
27
34
  body.size
28
35
  end
29
36
 
37
+ # @todo
30
38
  def mime
31
39
  content_type.split(";").first
32
40
  end
33
41
 
42
+ # @todo
34
43
  def charset
35
44
  content_type.include?("charset=") ? content_type.split("charset=").last : nil
36
45
  end
37
46
 
47
+ # @todo
38
48
  def to_text
39
- body
49
+ Text.new(body)
40
50
  end
41
51
 
52
+ # @todo
42
53
  def to_html
43
- HTML.new(body)
54
+ HTML.new(self)
44
55
  end
45
56
 
57
+ # @todo
46
58
  def to_xml
47
59
  XML.new(body)
48
60
  end
49
61
 
62
+ # @todo
50
63
  def to_json
51
64
  JSON.new(body)
52
65
  end
53
66
 
67
+ # @todo
54
68
  def to_markdown
55
69
  Markdown.new(to_html)
56
70
  end
57
71
 
72
+ # @todo
58
73
  def to_native
59
- @body
74
+ body
75
+ end
76
+
77
+ def to_s
78
+ to_native
60
79
  end
61
80
 
62
81
  private
63
82
 
83
+ # @todo
64
84
  def content_type
65
- @mech['content-type'] || ""
85
+ headers['content-type'] || ""
66
86
  end
67
87
 
68
88
  end