saper 0.5.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/LICENSE +21 -0
- data/README.md +126 -0
- data/Rakefile +17 -0
- data/bin/saper +60 -0
- data/lib/lib/json_search.rb +54 -0
- data/lib/lib/mechanize.rb +26 -0
- data/lib/lib/nokogiri.rb +12 -0
- data/lib/saper.rb +37 -0
- data/lib/saper/actions/append_with.rb +14 -0
- data/lib/saper/actions/convert_to_html.rb +14 -0
- data/lib/saper/actions/convert_to_json.rb +14 -0
- data/lib/saper/actions/convert_to_markdown.rb +13 -0
- data/lib/saper/actions/convert_to_time.rb +15 -0
- data/lib/saper/actions/convert_to_xml.rb +14 -0
- data/lib/saper/actions/create_atom.rb +18 -0
- data/lib/saper/actions/fetch.rb +17 -0
- data/lib/saper/actions/find.rb +18 -0
- data/lib/saper/actions/find_first.rb +16 -0
- data/lib/saper/actions/get_attribute.rb +15 -0
- data/lib/saper/actions/get_contents.rb +14 -0
- data/lib/saper/actions/get_text.rb +14 -0
- data/lib/saper/actions/prepend_with.rb +14 -0
- data/lib/saper/actions/remove_after.rb +14 -0
- data/lib/saper/actions/remove_before.rb +14 -0
- data/lib/saper/actions/remove_matching.rb +14 -0
- data/lib/saper/actions/remove_tags.rb +15 -0
- data/lib/saper/actions/replace.rb +15 -0
- data/lib/saper/actions/run_recipe.rb +24 -0
- data/lib/saper/actions/run_recipe_and_save.rb +22 -0
- data/lib/saper/actions/save.rb +14 -0
- data/lib/saper/actions/select_matching.rb +14 -0
- data/lib/saper/actions/set_input.rb +19 -0
- data/lib/saper/actions/skip_tags.rb +15 -0
- data/lib/saper/actions/split.rb +24 -0
- data/lib/saper/arguments/attribute.rb +11 -0
- data/lib/saper/arguments/recipe.rb +42 -0
- data/lib/saper/arguments/text.rb +11 -0
- data/lib/saper/arguments/timezone.rb +11 -0
- data/lib/saper/arguments/variable.rb +11 -0
- data/lib/saper/arguments/xpath.rb +11 -0
- data/lib/saper/core/action.rb +209 -0
- data/lib/saper/core/argument.rb +106 -0
- data/lib/saper/core/browser.rb +87 -0
- data/lib/saper/core/dsl.rb +68 -0
- data/lib/saper/core/error.rb +47 -0
- data/lib/saper/core/item.rb +70 -0
- data/lib/saper/core/keychain.rb +18 -0
- data/lib/saper/core/logger.rb +74 -0
- data/lib/saper/core/namespace.rb +139 -0
- data/lib/saper/core/recipe.rb +134 -0
- data/lib/saper/core/runtime.rb +237 -0
- data/lib/saper/core/type.rb +45 -0
- data/lib/saper/items/atom.rb +64 -0
- data/lib/saper/items/document.rb +66 -0
- data/lib/saper/items/html.rb +85 -0
- data/lib/saper/items/json.rb +67 -0
- data/lib/saper/items/markdown.rb +36 -0
- data/lib/saper/items/nothing.rb +15 -0
- data/lib/saper/items/text.rb +54 -0
- data/lib/saper/items/time.rb +42 -0
- data/lib/saper/items/url.rb +34 -0
- data/lib/saper/items/xml.rb +79 -0
- data/lib/saper/version.rb +3 -0
- data/spec/actions/append_with_spec.rb +30 -0
- data/spec/actions/convert_to_html_spec.rb +24 -0
- data/spec/actions/convert_to_json_spec.rb +24 -0
- data/spec/actions/convert_to_markdown_spec.rb +24 -0
- data/spec/actions/convert_to_time_spec.rb +37 -0
- data/spec/actions/convert_to_xml_spec.rb +24 -0
- data/spec/actions/create_atom_spec.rb +31 -0
- data/spec/actions/fetch_spec.rb +7 -0
- data/spec/actions/find_first_spec.rb +7 -0
- data/spec/actions/find_spec.rb +7 -0
- data/spec/actions/get_attribute_spec.rb +7 -0
- data/spec/actions/get_contents.rb +7 -0
- data/spec/actions/get_text.rb +7 -0
- data/spec/actions/prepend_with_spec.rb +30 -0
- data/spec/actions/remove_after.rb +7 -0
- data/spec/actions/remove_before.rb +7 -0
- data/spec/actions/replace_spec.rb +7 -0
- data/spec/actions/run_recipe_and_save_spec.tmp.rb +52 -0
- data/spec/actions/run_recipe_spec.tmp.rb +53 -0
- data/spec/actions/save_spec.rb +7 -0
- data/spec/actions/select_matching_spec.rb +7 -0
- data/spec/actions/set_input_spec.rb +7 -0
- data/spec/actions/skip_tags_spec.rb +7 -0
- data/spec/actions/split_spec.rb +7 -0
- data/spec/core/action_spec.rb +151 -0
- data/spec/core/argument_spec.rb +79 -0
- data/spec/core/browser_spec.rb +7 -0
- data/spec/core/dsl_spec.rb +7 -0
- data/spec/core/item_spec.rb +7 -0
- data/spec/core/keychain_spec.rb +7 -0
- data/spec/core/logger_spec.rb +7 -0
- data/spec/core/namespace_spec.rb +18 -0
- data/spec/core/recipe_spec.rb +81 -0
- data/spec/core/runtime_spec.rb +165 -0
- data/spec/core/type_spec.rb +7 -0
- data/spec/items/atom_spec.rb +7 -0
- data/spec/items/document_spec.rb +7 -0
- data/spec/items/html_spec.rb +7 -0
- data/spec/items/json_spec.rb +7 -0
- data/spec/items/markdown_spec.rb +7 -0
- data/spec/items/nothing_spec.rb +7 -0
- data/spec/items/text_spec.rb +17 -0
- data/spec/items/time_spec.rb +7 -0
- data/spec/items/url_spec.rb +7 -0
- data/spec/items/xml_spec.rb +17 -0
- data/spec/spec_helper.rb +22 -0
- metadata +355 -0
@@ -0,0 +1,18 @@
|
|
1
|
+
module Saper
|
2
|
+
module Actions
|
3
|
+
class Find < Action
|
4
|
+
|
5
|
+
returns_multiple_items!
|
6
|
+
argument :xpath
|
7
|
+
|
8
|
+
accepts :html, :returns => :html
|
9
|
+
accepts :xml, :returns => :xml
|
10
|
+
accepts :json, :returns => :json
|
11
|
+
|
12
|
+
run do |input, xpath|
|
13
|
+
input.find_all(xpath)
|
14
|
+
end
|
15
|
+
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
module Saper
|
2
|
+
module Actions
|
3
|
+
class FindFirst < Action
|
4
|
+
|
5
|
+
argument :xpath
|
6
|
+
accepts :html, :returns => :html
|
7
|
+
accepts :xml, :returns => :xml
|
8
|
+
accepts :json, :returns => :json
|
9
|
+
|
10
|
+
run do |input, xpath|
|
11
|
+
input.find(xpath)
|
12
|
+
end
|
13
|
+
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
module Saper
|
2
|
+
module Actions
|
3
|
+
class RunRecipe < Action
|
4
|
+
|
5
|
+
argument :recipe
|
6
|
+
accepts :anything, :returns => Proc.new { |instance| nil } # TODO
|
7
|
+
|
8
|
+
run do |input, subrecipe|
|
9
|
+
recipe(subrecipe, input)
|
10
|
+
end
|
11
|
+
|
12
|
+
# Overrides default logic
|
13
|
+
def multiple?
|
14
|
+
args.first.multiple?
|
15
|
+
end
|
16
|
+
|
17
|
+
# Overrides default logic
|
18
|
+
def requires
|
19
|
+
args.first.input_required
|
20
|
+
end
|
21
|
+
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
module Saper
|
2
|
+
module Actions
|
3
|
+
class RunRecipeAndSave < Action
|
4
|
+
|
5
|
+
argument :variable
|
6
|
+
argument :recipe
|
7
|
+
accepts :anything
|
8
|
+
|
9
|
+
run do |input, variable, subrecipe|
|
10
|
+
result = recipe(subrecipe, input)
|
11
|
+
self[variable] = result unless result.nil?
|
12
|
+
input
|
13
|
+
end
|
14
|
+
|
15
|
+
# Overrides default logic
|
16
|
+
def requires
|
17
|
+
args.last.input_required
|
18
|
+
end
|
19
|
+
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
module Saper
|
2
|
+
module Actions
|
3
|
+
class SkipTags < Action
|
4
|
+
|
5
|
+
argument :xpath
|
6
|
+
accepts :xml, :returns => :xml
|
7
|
+
accepts :html, :returns => :html
|
8
|
+
|
9
|
+
run do |input, xpath|
|
10
|
+
input.remove_children_preserving_content(xpath)
|
11
|
+
end
|
12
|
+
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
module Saper
|
2
|
+
module Actions
|
3
|
+
class Split < Action
|
4
|
+
|
5
|
+
returns_multiple_items!
|
6
|
+
argument :text
|
7
|
+
accepts :text, :returns => :text
|
8
|
+
accepts :document, :returns => :text
|
9
|
+
accepts :html, :returns => :html
|
10
|
+
|
11
|
+
run do |input, separator|
|
12
|
+
case input
|
13
|
+
when Items::HTML
|
14
|
+
input.inner_html.split(separator).map { |string| Item.new(:text, string).to_html }
|
15
|
+
when Items::Text
|
16
|
+
input.split(separator)
|
17
|
+
when Items::Document
|
18
|
+
input.to_text.split(separator)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
module Saper
|
2
|
+
module Arguments
|
3
|
+
class Recipe < Argument
|
4
|
+
|
5
|
+
def valid?(value)
|
6
|
+
value.is_a?(Symbol) || value.is_a?(String)
|
7
|
+
end
|
8
|
+
|
9
|
+
def normalize(value)
|
10
|
+
if value.is_a?(Recipe)
|
11
|
+
return value
|
12
|
+
end
|
13
|
+
if value.is_a?(String)
|
14
|
+
value = value.to_sym
|
15
|
+
end
|
16
|
+
unless value.is_a?(Symbol)
|
17
|
+
raise InvalidArgument
|
18
|
+
end
|
19
|
+
if action.nil?
|
20
|
+
raise NamespaceMissing
|
21
|
+
end
|
22
|
+
if action.namespace.nil?
|
23
|
+
raise NamespaceMissing
|
24
|
+
end
|
25
|
+
unless value = action.namespace[value]
|
26
|
+
raise ActionNotFound
|
27
|
+
end
|
28
|
+
value
|
29
|
+
end
|
30
|
+
|
31
|
+
#
|
32
|
+
def serialize
|
33
|
+
value.id
|
34
|
+
end
|
35
|
+
|
36
|
+
def to_string
|
37
|
+
value.id.inspect
|
38
|
+
end
|
39
|
+
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|