saper 0.5.1 → 0.5.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (120) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +3 -0
  3. data/Rakefile +6 -6
  4. data/bin/saper +17 -36
  5. data/lib/saper.rb +63 -20
  6. data/lib/saper/actions/append_with.rb +1 -1
  7. data/lib/saper/actions/convert_to_html.rb +1 -1
  8. data/lib/saper/actions/convert_to_json.rb +6 -2
  9. data/lib/saper/actions/convert_to_markdown.rb +1 -1
  10. data/lib/saper/actions/convert_to_time.rb +38 -1
  11. data/lib/saper/actions/convert_to_xml.rb +1 -1
  12. data/lib/saper/actions/create_atom.rb +6 -2
  13. data/lib/saper/actions/fetch.rb +3 -2
  14. data/lib/saper/actions/fetch_with_token.rb +18 -0
  15. data/lib/saper/actions/find.rb +1 -1
  16. data/lib/saper/actions/find_first.rb +1 -1
  17. data/lib/saper/actions/get_attribute.rb +7 -2
  18. data/lib/saper/actions/get_contents.rb +1 -1
  19. data/lib/saper/actions/get_text.rb +1 -1
  20. data/lib/saper/actions/nothing.rb +11 -0
  21. data/lib/saper/actions/prepend_with.rb +2 -2
  22. data/lib/saper/actions/remove_after.rb +2 -2
  23. data/lib/saper/actions/remove_before.rb +2 -2
  24. data/lib/saper/actions/remove_matching.rb +2 -2
  25. data/lib/saper/actions/remove_tags.rb +1 -1
  26. data/lib/saper/actions/replace.rb +1 -1
  27. data/lib/saper/actions/run_recipe.rb +2 -2
  28. data/lib/saper/actions/run_recipe_and_save.rb +3 -3
  29. data/lib/saper/actions/save.rb +2 -2
  30. data/lib/saper/actions/select_matching.rb +2 -2
  31. data/lib/saper/actions/set_input.rb +1 -1
  32. data/lib/saper/actions/skip_tags.rb +1 -1
  33. data/lib/saper/actions/split.rb +1 -1
  34. data/lib/saper/arguments/attribute.rb +6 -0
  35. data/lib/saper/arguments/recipe.rb +12 -6
  36. data/lib/saper/arguments/service.rb +12 -0
  37. data/lib/saper/arguments/text.rb +1 -0
  38. data/lib/saper/arguments/timezone.rb +2 -1
  39. data/lib/saper/arguments/url.rb +12 -0
  40. data/lib/saper/arguments/variable.rb +2 -1
  41. data/lib/saper/arguments/xpath.rb +2 -1
  42. data/lib/saper/core/action.rb +43 -13
  43. data/lib/saper/core/argument.rb +30 -14
  44. data/lib/saper/core/browser.rb +29 -28
  45. data/lib/saper/core/dsl.rb +14 -16
  46. data/lib/saper/core/error.rb +1 -1
  47. data/lib/saper/core/item.rb +2 -2
  48. data/lib/saper/core/keychain.rb +58 -5
  49. data/lib/saper/core/namespace.rb +15 -18
  50. data/lib/saper/core/recipe.rb +53 -12
  51. data/lib/saper/core/result.rb +72 -0
  52. data/lib/saper/core/runtime.rb +44 -177
  53. data/lib/saper/core/stack.rb +57 -0
  54. data/lib/saper/items/atom.rb +18 -1
  55. data/lib/saper/items/document.rb +39 -19
  56. data/lib/saper/items/html.rb +52 -7
  57. data/lib/saper/items/json.rb +19 -1
  58. data/lib/saper/items/markdown.rb +14 -4
  59. data/lib/saper/items/nothing.rb +6 -0
  60. data/lib/saper/items/text.rb +28 -3
  61. data/lib/saper/items/time.rb +25 -2
  62. data/lib/saper/items/url.rb +8 -2
  63. data/lib/saper/items/xml.rb +51 -11
  64. data/lib/{lib → saper/patches}/mechanize.rb +0 -0
  65. data/lib/{lib → saper/patches}/nokogiri.rb +0 -0
  66. data/lib/saper/version.rb +2 -2
  67. data/spec/actions/append_with_spec.rb +29 -52
  68. data/spec/actions/convert_to_html_spec.rb +13 -30
  69. data/spec/actions/convert_to_json_spec.rb +13 -30
  70. data/spec/actions/convert_to_markdown_spec.rb +17 -19
  71. data/spec/actions/convert_to_time_spec.rb +25 -43
  72. data/spec/actions/convert_to_xml_spec.rb +15 -11
  73. data/spec/actions/create_atom_spec.rb +11 -19
  74. data/spec/actions/fetch_spec.rb +3 -8
  75. data/spec/actions/find_first_spec.rb +38 -40
  76. data/spec/actions/find_spec.rb +23 -39
  77. data/spec/actions/get_attribute_spec.rb +30 -3
  78. data/spec/actions/{get_contents.rb → get_contents_spec.rb} +0 -0
  79. data/spec/actions/{get_text.rb → get_text_spec.rb} +0 -0
  80. data/spec/actions/nothing_spec.rb +7 -0
  81. data/spec/actions/prepend_with_spec.rb +31 -18
  82. data/spec/actions/{remove_after.rb → remove_after_spec.rb} +0 -0
  83. data/spec/actions/{remove_before.rb → remove_before_spec.rb} +0 -0
  84. data/spec/actions/remove_matching_spec.rb +7 -0
  85. data/spec/actions/remove_tags_spec.rb +7 -0
  86. data/spec/actions/run_recipe_and_save_spec.rb +7 -0
  87. data/spec/actions/run_recipe_spec.rb +7 -0
  88. data/spec/arguments/attribute_spec.rb +7 -0
  89. data/spec/arguments/recipe_spec.rb +7 -0
  90. data/spec/arguments/text_spec.rb +7 -0
  91. data/spec/arguments/timezone_spec.rb +7 -0
  92. data/spec/arguments/variable_spec.rb +7 -0
  93. data/spec/arguments/xpath_spec.rb +7 -0
  94. data/spec/core/action_spec.rb +13 -142
  95. data/spec/core/argument_spec.rb +38 -71
  96. data/spec/core/browser_spec.rb +18 -3
  97. data/spec/core/dsl_spec.rb +1 -1
  98. data/spec/core/item_spec.rb +1 -1
  99. data/spec/core/namespace_spec.rb +0 -11
  100. data/spec/core/recipe_spec.rb +54 -77
  101. data/spec/core/result_spec.rb +7 -0
  102. data/spec/core/runtime_spec.rb +20 -151
  103. data/spec/core/stack_spec.rb +7 -0
  104. data/spec/integration/simple_invalid_spec.rb +39 -0
  105. data/spec/integration/simple_valid_spec.rb +39 -0
  106. data/spec/items/atom_spec.rb +6 -1
  107. data/spec/items/document_spec.rb +93 -15
  108. data/spec/items/html_spec.rb +45 -28
  109. data/spec/items/json_spec.rb +10 -10
  110. data/spec/items/markdown_spec.rb +24 -3
  111. data/spec/items/nothing_spec.rb +1 -1
  112. data/spec/items/text_spec.rb +13 -41
  113. data/spec/items/time_spec.rb +25 -4
  114. data/spec/items/url_spec.rb +1 -7
  115. data/spec/items/xml_spec.rb +46 -39
  116. data/spec/spec_helper.rb +2 -21
  117. metadata +63 -60
  118. data/lib/lib/json_search.rb +0 -54
  119. data/spec/actions/run_recipe_and_save_spec.tmp.rb +0 -52
  120. data/spec/actions/run_recipe_spec.tmp.rb +0 -53
@@ -1,52 +1,51 @@
1
1
  module Saper
2
+ # Runtime provides a set of auxiliary functions to Saper actions:
3
+ # browser requests, keychain access, storage of variables.
2
4
  class Runtime
3
5
 
4
- # Returns action input.
5
- attr_reader :input
6
-
7
- # Returns action output.
8
- attr_reader :output
9
-
10
- # Returns error instance (if any)
11
- attr_reader :error
12
-
13
- # Returns action instance.
14
- attr_reader :action
15
-
16
- # Returns embedded recipes
17
- attr_reader :subrecipes
6
+ # Returns Saper::Browser instance used by runtime.
7
+ # @return [Saper::Browser]
8
+ attr_reader :browser
18
9
 
19
- # Returns subsequent runtime instances.
20
- attr_reader :children
10
+ # Returns Saper::Keychain instance used by runtime.
11
+ # @return [Saper::Keychain]
12
+ attr_reader :keychain
21
13
 
22
- # Returns the size of action chain.
23
- attr_reader :depth
14
+ # Returns a Hash of variables stored by runtime.
15
+ # @return [Hash]
16
+ attr_reader :variables
24
17
 
25
18
  # Returns a new Saper::Runtime instance.
26
- # @param stack [Array<Saper::Action>] chain of actions
27
- # @param input [Object, nil] action input
19
+ # @param variables [Hash]
28
20
  # @param options [Hash]
21
+ # @option options [Browser] :browser Browser instance
22
+ # @option options [String, Keychain] :keychain Keychain file or instance
23
+ # @option options [String, Symbol] :agent User-Agent type or string
29
24
  # @return [Saper::Runtime]
30
- def initialize(stack = [], input = nil, options = {})
31
- if stack.is_a?(Recipe)
32
- stack = stack.actions
25
+ def initialize(variables = {}, options = {})
26
+ if options.nil?
27
+ puts self.inspect
28
+ raise 'stop'
29
+ end
30
+ if options[:keychain].is_a?(Keychain)
31
+ @keychain = options[:keychain]
32
+ end
33
+ if options[:keychain].is_a?(String)
34
+ @keychain = Keychain.load(options[:keychain])
35
+ end
36
+ unless @keychain.is_a?(Keychain)
37
+ @keychain = Keychain.new
33
38
  end
34
- @input = input
35
- @output = nil
36
- @error = nil
37
- @action = nil
38
- @children = []
39
- @subrecipes = []
40
- @options = options
41
- @depth = stack.size
42
- @options[:keychain] ||= Keychain.new
43
- @options[:logger] ||= Logger.new(@options[:log])
44
- @options[:browser] ||= Browser.new(:logger => logger)
45
- @options[:variables] ||= {}
39
+ if options[:browser].is_a?(Browser)
40
+ @browser = options[:browser]
41
+ end
42
+ unless @browser.is_a?(Browser)
43
+ @browser = Browser.new(:agent => options[:agent])
44
+ end
45
+ @variables = variables
46
46
  if block_given?
47
47
  yield self
48
48
  end
49
- execute(stack.dup)
50
49
  end
51
50
 
52
51
  # Returns volume of incoming traffic (in bytes)
@@ -71,7 +70,7 @@ module Saper
71
70
  # @param name [Symbol]
72
71
  # @return [Object]
73
72
  def [](name)
74
- @options[:variables][name]
73
+ @variables[name]
75
74
  end
76
75
 
77
76
  # Set variable value
@@ -79,62 +78,7 @@ module Saper
79
78
  # @param value [Object]
80
79
  # @return [Object]
81
80
  def []=(name, value)
82
- @options[:variables][name] = value
83
- end
84
-
85
- # Return a hash with all variables
86
- # @return [Hash]
87
- def variables
88
- @options[:variables].dup
89
- end
90
-
91
- # Returns descendant runtime instances at a given depth level.
92
- # @param level [Integer]
93
- # @return [Array<Saper::Runtime>]
94
- def descendants(level = 0)
95
- level < 1 ? [self] : children.map { |i| i.descendants(level - 1) }.flatten
96
- end
97
-
98
- # Returns runtime results.
99
- # @return [Array<Object>, Object]
100
- def results
101
- native_results
102
- end
103
-
104
- # Returns an array of results.
105
- # @return [Array]
106
- def result_array
107
- native_result_array
108
- end
109
-
110
- # Returns `true` if this or any of the subsequent actions produces multiple results.
111
- # @return [Boolean]
112
- def multiple?
113
- (action.nil? ? false : action.multiple?) || children.any?(&:multiple?)
114
- end
115
-
116
- # Returns Saper::Browser instance used by runtime.
117
- # @return [Saper::Browser]
118
- def browser
119
- @options[:browser]
120
- end
121
-
122
- # Returns Saper::Keychain instance used by runtime.
123
- # @return [Saper::Keychain]
124
- def keychain
125
- @options[:keychain]
126
- end
127
-
128
- # Returns Saper::Logger.
129
- # @return [Saper::Logger]
130
- def logger
131
- @options[:logger]
132
- end
133
-
134
- # Writes full runtime log to logger instance.
135
- # @return [void]
136
- def backtrace
137
- logger.runtime(self)
81
+ @variables[name] = value
138
82
  end
139
83
 
140
84
  # Returns access credentials for specified service
@@ -144,93 +88,16 @@ module Saper
144
88
  keychain[service]
145
89
  end
146
90
 
147
- # Runs an arbitrary recipe, specified by ID, block or as instance.
148
- # @param recipe [Saper::Recipe]
149
- # @param input [Object, nil]
91
+ # Returns a duplicate instance of self.
150
92
  # @return [Saper::Runtime]
151
- def recipe(recipe, input = nil, &block)
152
- unless recipe.is_a?(Recipe)
153
- raise RecipeNotFound, recipe
154
- end
155
- runtime = Runtime.new(recipe, input, @options.merge(:variables => {}))
156
- if runtime.error?
157
- @error = runtime.error
158
- end
159
- subrecipes << runtime
160
- runtime.non_native_results
161
- end
162
-
163
- # Returns `true` if action raised no errors.
164
- # @return [Boolean]
165
- def success?
166
- error.nil?
167
- end
168
-
169
- # Returns `true` if action raised at least one error.
170
- # @return [Boolean]
171
- def error?
172
- !success?
173
- end
174
-
175
- # Returns runtime results as JSON.
176
- # @return [Array]
177
- def to_json(*args)
178
- JSON.generate non_native_results.map(&:serialize)
179
- end
180
-
181
- # Returns runtime results as Saper::Item instances.
182
- # @return [Saper::Item, Array<Saper::Item>]
183
- def non_native_results
184
- multiple? ? non_native_result_array : non_native_result_array.first
185
- end
186
-
187
- # Returns runtime results as native Ruby objects.
188
- # @return [Object, Array<Object>]
189
- def native_results
190
- multiple? ? native_result_array : native_result_array.first
191
- end
192
-
193
- private
194
-
195
- # Returns runtime results as an array of Saper::Item instances.
196
- # @return [Array<Saper::Item>]
197
- def non_native_result_array
198
- descendants(depth - 1).map(&:output).flatten.compact
199
- end
200
-
201
- # Returns runtime results as an array of native Ruby objects.
202
- # @return [Array<Object>]
203
- def native_result_array
204
- non_native_result_array.map { |result| result.is_a?(Item) ? result.to_native : result }
93
+ def copy
94
+ Runtime.new(variables.dup, :browser => browser, :keychain => keychain)
205
95
  end
206
96
 
207
- # Executes recipes.
208
- # @return [self]
209
- def execute(stack)
210
- @output = input
211
- @action = stack.shift
212
- if action.nil?
213
- return self
214
- end
215
- unless action.is_a?(Saper::Action)
216
- raise ActionExpected, action
217
- end
218
- begin
219
- @output = action.run(input, self)
220
- rescue Saper::Error => e
221
- @output = nil
222
- @error = e
223
- end
224
- unless @output.is_a?(Array)
225
- @output = [@output]
226
- end
227
- if stack.empty?
228
- return self
229
- end
230
- @children = @output.map do |item|
231
- Runtime.new(stack, item, @options.dup)
232
- end
233
- self
97
+ # Returns a duplicate instance of self without the stored variables.
98
+ # @return [Saper::Runtime]
99
+ def copy_without_variables
100
+ Runtime.new({}, :browser => browser, :keychain => keychain)
234
101
  end
235
102
 
236
103
  end
@@ -0,0 +1,57 @@
1
+ module Saper
2
+ class Stack
3
+
4
+ # Returns Runtime instance used by the stack
5
+ # @return [Saper::Runtime]
6
+ attr_reader :runtime
7
+
8
+ # Returns all items in the stack.
9
+ # @return [Array<Saper::Item>]
10
+ attr_reader :items
11
+
12
+ # Returns a new Stack instance.
13
+ # @param runtime [Saper::Runtime]
14
+ # @param items [Saper::Item]
15
+ # @return [Saper::Stack]
16
+ def initialize(runtime, *items)
17
+ @runtime, @items = runtime, items.flatten
18
+ end
19
+
20
+ # Returns a duplicate instance of self.
21
+ # @return [Saper::Stack]
22
+ def copy
23
+ Stack.new(runtime.copy, *to_a)
24
+ end
25
+
26
+ # Adds action result to the end of the stack and returns self.
27
+ # @param item [Saper::Item]
28
+ # @return [self]
29
+ def add(item)
30
+ @items.push(item); self
31
+ end
32
+
33
+ # Returns the result of the last action or nil (in case of an error).
34
+ # @return [Saper::Item]
35
+ def result
36
+ error? ? Items::Nothing.new : last
37
+ end
38
+
39
+ # Returns the result of the last action.
40
+ # @return [Saper::Item]
41
+ def last
42
+ @items.last || Items::Nothing.new
43
+ end
44
+
45
+ # @todo
46
+ def error?
47
+ last.is_a?(Saper::Error)
48
+ end
49
+
50
+ # Returns an array of action results.
51
+ # @return [Array<Saper::Item>]
52
+ def to_a
53
+ @items.dup
54
+ end
55
+
56
+ end
57
+ end
@@ -2,6 +2,7 @@ module Saper
2
2
  module Items
3
3
  class Atom < Item
4
4
 
5
+ # @todo
5
6
  def self.new(item)
6
7
  super case item
7
8
  when Hash
@@ -11,22 +12,32 @@ module Saper
11
12
  end
12
13
  end
13
14
 
15
+ # @todo
14
16
  def initialize(hash)
15
17
  @atts = hash
16
18
  end
17
19
 
20
+ # @todo
21
+ def ==(other)
22
+ to_hash == other
23
+ end
24
+
25
+ # @todo
18
26
  def to_hash
19
27
  @atts.dup
20
28
  end
21
29
 
30
+ # @todo
22
31
  def [](name)
23
32
  @atts[name]
24
33
  end
25
34
 
35
+ # @todo
26
36
  def serialize
27
- @atts.dup
37
+ Hash[@atts.map { |key, value| [key, value.respond_to?(:serialize) ? value.serialize : value] }]
28
38
  end
29
39
 
40
+ # @todo
30
41
  def to_native(object = nil)
31
42
  if object.nil?
32
43
  return to_native(to_hash)
@@ -43,10 +54,16 @@ module Saper
43
54
  object
44
55
  end
45
56
 
57
+ # @todo
46
58
  def to_json
47
59
  JSON.new(@hash, true)
48
60
  end
49
61
 
62
+ # @todo
63
+ def to_s
64
+ to_json
65
+ end
66
+
50
67
  end
51
68
  end
52
69
  end
@@ -2,67 +2,87 @@ module Saper
2
2
  module Items
3
3
  class Document < Item
4
4
 
5
- def self.new(item)
6
- super case item
7
- when Mechanize::File
8
- item
9
- else
10
- raise(InvalidItem, item)
11
- end
12
- end
5
+ # @todo
6
+ attr_reader :body
13
7
 
14
- def initialize(mechanize)
15
- @mech = mechanize
16
- end
8
+ # @todo
9
+ attr_reader :uri
17
10
 
18
- def uri
19
- @mech.uri.to_s
11
+ # @todo
12
+ attr_reader :headers
13
+
14
+ # @todo
15
+ def self.new(body, uri = nil, headers = {})
16
+ unless body.is_a?(String)
17
+ raise(InvalidItem)
18
+ end
19
+ unless headers.is_a?(Hash)
20
+ raise(InvalidItem)
21
+ end
22
+ super
20
23
  end
21
24
 
22
- def body
23
- @mech.body
25
+ # @todo
26
+ def initialize(body, uri = nil, headers = {})
27
+ @uri = (uri.nil? ? nil : uri.to_s)
28
+ @body = body
29
+ @headers = headers
24
30
  end
25
31
 
32
+ # @todo
26
33
  def size
27
34
  body.size
28
35
  end
29
36
 
37
+ # @todo
30
38
  def mime
31
39
  content_type.split(";").first
32
40
  end
33
41
 
42
+ # @todo
34
43
  def charset
35
44
  content_type.include?("charset=") ? content_type.split("charset=").last : nil
36
45
  end
37
46
 
47
+ # @todo
38
48
  def to_text
39
- body
49
+ Text.new(body)
40
50
  end
41
51
 
52
+ # @todo
42
53
  def to_html
43
- HTML.new(body)
54
+ HTML.new(self)
44
55
  end
45
56
 
57
+ # @todo
46
58
  def to_xml
47
59
  XML.new(body)
48
60
  end
49
61
 
62
+ # @todo
50
63
  def to_json
51
64
  JSON.new(body)
52
65
  end
53
66
 
67
+ # @todo
54
68
  def to_markdown
55
69
  Markdown.new(to_html)
56
70
  end
57
71
 
72
+ # @todo
58
73
  def to_native
59
- @body
74
+ body
75
+ end
76
+
77
+ def to_s
78
+ to_native
60
79
  end
61
80
 
62
81
  private
63
82
 
83
+ # @todo
64
84
  def content_type
65
- @mech['content-type'] || ""
85
+ headers['content-type'] || ""
66
86
  end
67
87
 
68
88
  end