saper 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (111) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +21 -0
  3. data/README.md +126 -0
  4. data/Rakefile +17 -0
  5. data/bin/saper +60 -0
  6. data/lib/lib/json_search.rb +54 -0
  7. data/lib/lib/mechanize.rb +26 -0
  8. data/lib/lib/nokogiri.rb +12 -0
  9. data/lib/saper.rb +37 -0
  10. data/lib/saper/actions/append_with.rb +14 -0
  11. data/lib/saper/actions/convert_to_html.rb +14 -0
  12. data/lib/saper/actions/convert_to_json.rb +14 -0
  13. data/lib/saper/actions/convert_to_markdown.rb +13 -0
  14. data/lib/saper/actions/convert_to_time.rb +15 -0
  15. data/lib/saper/actions/convert_to_xml.rb +14 -0
  16. data/lib/saper/actions/create_atom.rb +18 -0
  17. data/lib/saper/actions/fetch.rb +17 -0
  18. data/lib/saper/actions/find.rb +18 -0
  19. data/lib/saper/actions/find_first.rb +16 -0
  20. data/lib/saper/actions/get_attribute.rb +15 -0
  21. data/lib/saper/actions/get_contents.rb +14 -0
  22. data/lib/saper/actions/get_text.rb +14 -0
  23. data/lib/saper/actions/prepend_with.rb +14 -0
  24. data/lib/saper/actions/remove_after.rb +14 -0
  25. data/lib/saper/actions/remove_before.rb +14 -0
  26. data/lib/saper/actions/remove_matching.rb +14 -0
  27. data/lib/saper/actions/remove_tags.rb +15 -0
  28. data/lib/saper/actions/replace.rb +15 -0
  29. data/lib/saper/actions/run_recipe.rb +24 -0
  30. data/lib/saper/actions/run_recipe_and_save.rb +22 -0
  31. data/lib/saper/actions/save.rb +14 -0
  32. data/lib/saper/actions/select_matching.rb +14 -0
  33. data/lib/saper/actions/set_input.rb +19 -0
  34. data/lib/saper/actions/skip_tags.rb +15 -0
  35. data/lib/saper/actions/split.rb +24 -0
  36. data/lib/saper/arguments/attribute.rb +11 -0
  37. data/lib/saper/arguments/recipe.rb +42 -0
  38. data/lib/saper/arguments/text.rb +11 -0
  39. data/lib/saper/arguments/timezone.rb +11 -0
  40. data/lib/saper/arguments/variable.rb +11 -0
  41. data/lib/saper/arguments/xpath.rb +11 -0
  42. data/lib/saper/core/action.rb +209 -0
  43. data/lib/saper/core/argument.rb +106 -0
  44. data/lib/saper/core/browser.rb +87 -0
  45. data/lib/saper/core/dsl.rb +68 -0
  46. data/lib/saper/core/error.rb +47 -0
  47. data/lib/saper/core/item.rb +70 -0
  48. data/lib/saper/core/keychain.rb +18 -0
  49. data/lib/saper/core/logger.rb +74 -0
  50. data/lib/saper/core/namespace.rb +139 -0
  51. data/lib/saper/core/recipe.rb +134 -0
  52. data/lib/saper/core/runtime.rb +237 -0
  53. data/lib/saper/core/type.rb +45 -0
  54. data/lib/saper/items/atom.rb +64 -0
  55. data/lib/saper/items/document.rb +66 -0
  56. data/lib/saper/items/html.rb +85 -0
  57. data/lib/saper/items/json.rb +67 -0
  58. data/lib/saper/items/markdown.rb +36 -0
  59. data/lib/saper/items/nothing.rb +15 -0
  60. data/lib/saper/items/text.rb +54 -0
  61. data/lib/saper/items/time.rb +42 -0
  62. data/lib/saper/items/url.rb +34 -0
  63. data/lib/saper/items/xml.rb +79 -0
  64. data/lib/saper/version.rb +3 -0
  65. data/spec/actions/append_with_spec.rb +30 -0
  66. data/spec/actions/convert_to_html_spec.rb +24 -0
  67. data/spec/actions/convert_to_json_spec.rb +24 -0
  68. data/spec/actions/convert_to_markdown_spec.rb +24 -0
  69. data/spec/actions/convert_to_time_spec.rb +37 -0
  70. data/spec/actions/convert_to_xml_spec.rb +24 -0
  71. data/spec/actions/create_atom_spec.rb +31 -0
  72. data/spec/actions/fetch_spec.rb +7 -0
  73. data/spec/actions/find_first_spec.rb +7 -0
  74. data/spec/actions/find_spec.rb +7 -0
  75. data/spec/actions/get_attribute_spec.rb +7 -0
  76. data/spec/actions/get_contents.rb +7 -0
  77. data/spec/actions/get_text.rb +7 -0
  78. data/spec/actions/prepend_with_spec.rb +30 -0
  79. data/spec/actions/remove_after.rb +7 -0
  80. data/spec/actions/remove_before.rb +7 -0
  81. data/spec/actions/replace_spec.rb +7 -0
  82. data/spec/actions/run_recipe_and_save_spec.tmp.rb +52 -0
  83. data/spec/actions/run_recipe_spec.tmp.rb +53 -0
  84. data/spec/actions/save_spec.rb +7 -0
  85. data/spec/actions/select_matching_spec.rb +7 -0
  86. data/spec/actions/set_input_spec.rb +7 -0
  87. data/spec/actions/skip_tags_spec.rb +7 -0
  88. data/spec/actions/split_spec.rb +7 -0
  89. data/spec/core/action_spec.rb +151 -0
  90. data/spec/core/argument_spec.rb +79 -0
  91. data/spec/core/browser_spec.rb +7 -0
  92. data/spec/core/dsl_spec.rb +7 -0
  93. data/spec/core/item_spec.rb +7 -0
  94. data/spec/core/keychain_spec.rb +7 -0
  95. data/spec/core/logger_spec.rb +7 -0
  96. data/spec/core/namespace_spec.rb +18 -0
  97. data/spec/core/recipe_spec.rb +81 -0
  98. data/spec/core/runtime_spec.rb +165 -0
  99. data/spec/core/type_spec.rb +7 -0
  100. data/spec/items/atom_spec.rb +7 -0
  101. data/spec/items/document_spec.rb +7 -0
  102. data/spec/items/html_spec.rb +7 -0
  103. data/spec/items/json_spec.rb +7 -0
  104. data/spec/items/markdown_spec.rb +7 -0
  105. data/spec/items/nothing_spec.rb +7 -0
  106. data/spec/items/text_spec.rb +17 -0
  107. data/spec/items/time_spec.rb +7 -0
  108. data/spec/items/url_spec.rb +7 -0
  109. data/spec/items/xml_spec.rb +17 -0
  110. data/spec/spec_helper.rb +22 -0
  111. metadata +355 -0
@@ -0,0 +1,14 @@
1
+ module Saper
2
+ module Actions
3
+ class ConvertToXML < Action
4
+
5
+ accepts :text, :returns => :xml
6
+ accepts :document, :returns => :xml
7
+
8
+ run do |input|
9
+ input.to_xml
10
+ end
11
+
12
+ end
13
+ end
14
+ end
@@ -0,0 +1,18 @@
1
+ module Saper
2
+ module Actions
3
+ class CreateAtom < Action
4
+
5
+ accepts :anything, :returns => :atom
6
+
7
+ run do |input|
8
+ Items::Atom.new(variables)
9
+ end
10
+
11
+ # Overrides default logic
12
+ def requires
13
+ []
14
+ end
15
+
16
+ end
17
+ end
18
+ end
@@ -0,0 +1,17 @@
1
+ module Saper
2
+ module Actions
3
+ class Fetch < Action
4
+
5
+ accepts :url, :returns => :document
6
+
7
+ run do |input|
8
+ begin
9
+ browser.get(input)
10
+ rescue SocketError
11
+ raise(UnreachableURL, input)
12
+ end
13
+ end
14
+
15
+ end
16
+ end
17
+ end
@@ -0,0 +1,18 @@
1
+ module Saper
2
+ module Actions
3
+ class Find < Action
4
+
5
+ returns_multiple_items!
6
+ argument :xpath
7
+
8
+ accepts :html, :returns => :html
9
+ accepts :xml, :returns => :xml
10
+ accepts :json, :returns => :json
11
+
12
+ run do |input, xpath|
13
+ input.find_all(xpath)
14
+ end
15
+
16
+ end
17
+ end
18
+ end
@@ -0,0 +1,16 @@
1
+ module Saper
2
+ module Actions
3
+ class FindFirst < Action
4
+
5
+ argument :xpath
6
+ accepts :html, :returns => :html
7
+ accepts :xml, :returns => :xml
8
+ accepts :json, :returns => :json
9
+
10
+ run do |input, xpath|
11
+ input.find(xpath)
12
+ end
13
+
14
+ end
15
+ end
16
+ end
@@ -0,0 +1,15 @@
1
+ module Saper
2
+ module Actions
3
+ class GetAttribute < Action
4
+
5
+ argument :text
6
+ accepts :xml, :returns => :text
7
+ accepts :html, :returns => :text
8
+
9
+ run do |input, name|
10
+ input[name]
11
+ end
12
+
13
+ end
14
+ end
15
+ end
@@ -0,0 +1,14 @@
1
+ module Saper
2
+ module Actions
3
+ class GetContents < Action
4
+
5
+ accepts :xml, :returns => :text
6
+ accepts :html, :returns => :text
7
+
8
+ run do |input|
9
+ input.inner_html
10
+ end
11
+
12
+ end
13
+ end
14
+ end
@@ -0,0 +1,14 @@
1
+ module Saper
2
+ module Actions
3
+ class GetText < Action
4
+
5
+ accepts :xml, :returns => :text
6
+ accepts :html, :returns => :text
7
+
8
+ run do |input|
9
+ input.inner_text
10
+ end
11
+
12
+ end
13
+ end
14
+ end
@@ -0,0 +1,14 @@
1
+ module Saper
2
+ module Actions
3
+ class PrependWith < Action
4
+
5
+ argument :text
6
+ accepts :text, :returns => :text
7
+
8
+ run do |input, string|
9
+ "%s%s" % [string, input]
10
+ end
11
+
12
+ end
13
+ end
14
+ end
@@ -0,0 +1,14 @@
1
+ module Saper
2
+ module Actions
3
+ class RemoveAfter < Action
4
+
5
+ argument :text
6
+ accepts :text, :returns => :text
7
+
8
+ run do |input, separator|
9
+ input.split(separator, 2).first
10
+ end
11
+
12
+ end
13
+ end
14
+ end
@@ -0,0 +1,14 @@
1
+ module Saper
2
+ module Actions
3
+ class RemoveBefore < Action
4
+
5
+ argument :text
6
+ accepts :text, :returns => :text
7
+
8
+ run do |input, separator|
9
+ input.split(separator, 2).last
10
+ end
11
+
12
+ end
13
+ end
14
+ end
@@ -0,0 +1,14 @@
1
+ module Saper
2
+ module Actions
3
+ class RemoveMatching < Action
4
+
5
+ argument :text
6
+ accepts :text, :returns => :text
7
+
8
+ run do |input, pattern|
9
+ input.to_s =~ Regexp.new(pattern) ? nil : input
10
+ end
11
+
12
+ end
13
+ end
14
+ end
@@ -0,0 +1,15 @@
1
+ module Saper
2
+ module Actions
3
+ class RemoveTags < Action
4
+
5
+ argument :xpath
6
+ accepts :xml, :returns => :xml
7
+ accepts :html, :returns => :html
8
+
9
+ run do |input, xpath|
10
+ input.remove_children_with_content(xpath)
11
+ end
12
+
13
+ end
14
+ end
15
+ end
@@ -0,0 +1,15 @@
1
+ module Saper
2
+ module Actions
3
+ class Replace < Action
4
+
5
+ argument :text
6
+ argument :text
7
+ accepts :text, :returns => :text
8
+
9
+ run do |input, what, with|
10
+ input.gsub(what, with)
11
+ end
12
+
13
+ end
14
+ end
15
+ end
@@ -0,0 +1,24 @@
1
+ module Saper
2
+ module Actions
3
+ class RunRecipe < Action
4
+
5
+ argument :recipe
6
+ accepts :anything, :returns => Proc.new { |instance| nil } # TODO
7
+
8
+ run do |input, subrecipe|
9
+ recipe(subrecipe, input)
10
+ end
11
+
12
+ # Overrides default logic
13
+ def multiple?
14
+ args.first.multiple?
15
+ end
16
+
17
+ # Overrides default logic
18
+ def requires
19
+ args.first.input_required
20
+ end
21
+
22
+ end
23
+ end
24
+ end
@@ -0,0 +1,22 @@
1
+ module Saper
2
+ module Actions
3
+ class RunRecipeAndSave < Action
4
+
5
+ argument :variable
6
+ argument :recipe
7
+ accepts :anything
8
+
9
+ run do |input, variable, subrecipe|
10
+ result = recipe(subrecipe, input)
11
+ self[variable] = result unless result.nil?
12
+ input
13
+ end
14
+
15
+ # Overrides default logic
16
+ def requires
17
+ args.last.input_required
18
+ end
19
+
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,14 @@
1
+ module Saper
2
+ module Actions
3
+ class Save < Action
4
+
5
+ argument :variable
6
+ accepts :anything
7
+
8
+ run do |input, variable|
9
+ self[variable] = input
10
+ end
11
+
12
+ end
13
+ end
14
+ end
@@ -0,0 +1,14 @@
1
+ module Saper
2
+ module Actions
3
+ class SelectMatching < Action
4
+
5
+ argument :text
6
+ accepts :text, :returns => :text
7
+
8
+ run do |input, pattern|
9
+ input.to_s =~ Regexp.new(pattern) ? input : nil
10
+ end
11
+
12
+ end
13
+ end
14
+ end
@@ -0,0 +1,19 @@
1
+ module Saper
2
+ module Actions
3
+ class SetInput < Action
4
+
5
+ argument :text
6
+ accepts :anything, :returns => :text
7
+
8
+ run do |input, string|
9
+ string
10
+ end
11
+
12
+ # Overrides default logic
13
+ def requires
14
+ []
15
+ end
16
+
17
+ end
18
+ end
19
+ end
@@ -0,0 +1,15 @@
1
+ module Saper
2
+ module Actions
3
+ class SkipTags < Action
4
+
5
+ argument :xpath
6
+ accepts :xml, :returns => :xml
7
+ accepts :html, :returns => :html
8
+
9
+ run do |input, xpath|
10
+ input.remove_children_preserving_content(xpath)
11
+ end
12
+
13
+ end
14
+ end
15
+ end
@@ -0,0 +1,24 @@
1
+ module Saper
2
+ module Actions
3
+ class Split < Action
4
+
5
+ returns_multiple_items!
6
+ argument :text
7
+ accepts :text, :returns => :text
8
+ accepts :document, :returns => :text
9
+ accepts :html, :returns => :html
10
+
11
+ run do |input, separator|
12
+ case input
13
+ when Items::HTML
14
+ input.inner_html.split(separator).map { |string| Item.new(:text, string).to_html }
15
+ when Items::Text
16
+ input.split(separator)
17
+ when Items::Document
18
+ input.to_text.split(separator)
19
+ end
20
+ end
21
+
22
+ end
23
+ end
24
+ end
@@ -0,0 +1,11 @@
1
+ module Saper
2
+ module Arguments
3
+ class Attribute < Argument
4
+
5
+ def valid?(value)
6
+ value.is_a?(Symbol) || value.is_a?(String)
7
+ end
8
+
9
+ end
10
+ end
11
+ end
@@ -0,0 +1,42 @@
1
+ module Saper
2
+ module Arguments
3
+ class Recipe < Argument
4
+
5
+ def valid?(value)
6
+ value.is_a?(Symbol) || value.is_a?(String)
7
+ end
8
+
9
+ def normalize(value)
10
+ if value.is_a?(Recipe)
11
+ return value
12
+ end
13
+ if value.is_a?(String)
14
+ value = value.to_sym
15
+ end
16
+ unless value.is_a?(Symbol)
17
+ raise InvalidArgument
18
+ end
19
+ if action.nil?
20
+ raise NamespaceMissing
21
+ end
22
+ if action.namespace.nil?
23
+ raise NamespaceMissing
24
+ end
25
+ unless value = action.namespace[value]
26
+ raise ActionNotFound
27
+ end
28
+ value
29
+ end
30
+
31
+ #
32
+ def serialize
33
+ value.id
34
+ end
35
+
36
+ def to_string
37
+ value.id.inspect
38
+ end
39
+
40
+ end
41
+ end
42
+ end
@@ -0,0 +1,11 @@
1
+ module Saper
2
+ module Arguments
3
+ class Text < Argument
4
+
5
+ def valid?(value)
6
+ value.is_a?(String)
7
+ end
8
+
9
+ end
10
+ end
11
+ end