saper 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +21 -0
- data/README.md +126 -0
- data/Rakefile +17 -0
- data/bin/saper +60 -0
- data/lib/lib/json_search.rb +54 -0
- data/lib/lib/mechanize.rb +26 -0
- data/lib/lib/nokogiri.rb +12 -0
- data/lib/saper.rb +37 -0
- data/lib/saper/actions/append_with.rb +14 -0
- data/lib/saper/actions/convert_to_html.rb +14 -0
- data/lib/saper/actions/convert_to_json.rb +14 -0
- data/lib/saper/actions/convert_to_markdown.rb +13 -0
- data/lib/saper/actions/convert_to_time.rb +15 -0
- data/lib/saper/actions/convert_to_xml.rb +14 -0
- data/lib/saper/actions/create_atom.rb +18 -0
- data/lib/saper/actions/fetch.rb +17 -0
- data/lib/saper/actions/find.rb +18 -0
- data/lib/saper/actions/find_first.rb +16 -0
- data/lib/saper/actions/get_attribute.rb +15 -0
- data/lib/saper/actions/get_contents.rb +14 -0
- data/lib/saper/actions/get_text.rb +14 -0
- data/lib/saper/actions/prepend_with.rb +14 -0
- data/lib/saper/actions/remove_after.rb +14 -0
- data/lib/saper/actions/remove_before.rb +14 -0
- data/lib/saper/actions/remove_matching.rb +14 -0
- data/lib/saper/actions/remove_tags.rb +15 -0
- data/lib/saper/actions/replace.rb +15 -0
- data/lib/saper/actions/run_recipe.rb +24 -0
- data/lib/saper/actions/run_recipe_and_save.rb +22 -0
- data/lib/saper/actions/save.rb +14 -0
- data/lib/saper/actions/select_matching.rb +14 -0
- data/lib/saper/actions/set_input.rb +19 -0
- data/lib/saper/actions/skip_tags.rb +15 -0
- data/lib/saper/actions/split.rb +24 -0
- data/lib/saper/arguments/attribute.rb +11 -0
- data/lib/saper/arguments/recipe.rb +42 -0
- data/lib/saper/arguments/text.rb +11 -0
- data/lib/saper/arguments/timezone.rb +11 -0
- data/lib/saper/arguments/variable.rb +11 -0
- data/lib/saper/arguments/xpath.rb +11 -0
- data/lib/saper/core/action.rb +209 -0
- data/lib/saper/core/argument.rb +106 -0
- data/lib/saper/core/browser.rb +87 -0
- data/lib/saper/core/dsl.rb +68 -0
- data/lib/saper/core/error.rb +47 -0
- data/lib/saper/core/item.rb +70 -0
- data/lib/saper/core/keychain.rb +18 -0
- data/lib/saper/core/logger.rb +74 -0
- data/lib/saper/core/namespace.rb +139 -0
- data/lib/saper/core/recipe.rb +134 -0
- data/lib/saper/core/runtime.rb +237 -0
- data/lib/saper/core/type.rb +45 -0
- data/lib/saper/items/atom.rb +64 -0
- data/lib/saper/items/document.rb +66 -0
- data/lib/saper/items/html.rb +85 -0
- data/lib/saper/items/json.rb +67 -0
- data/lib/saper/items/markdown.rb +36 -0
- data/lib/saper/items/nothing.rb +15 -0
- data/lib/saper/items/text.rb +54 -0
- data/lib/saper/items/time.rb +42 -0
- data/lib/saper/items/url.rb +34 -0
- data/lib/saper/items/xml.rb +79 -0
- data/lib/saper/version.rb +3 -0
- data/spec/actions/append_with_spec.rb +30 -0
- data/spec/actions/convert_to_html_spec.rb +24 -0
- data/spec/actions/convert_to_json_spec.rb +24 -0
- data/spec/actions/convert_to_markdown_spec.rb +24 -0
- data/spec/actions/convert_to_time_spec.rb +37 -0
- data/spec/actions/convert_to_xml_spec.rb +24 -0
- data/spec/actions/create_atom_spec.rb +31 -0
- data/spec/actions/fetch_spec.rb +7 -0
- data/spec/actions/find_first_spec.rb +7 -0
- data/spec/actions/find_spec.rb +7 -0
- data/spec/actions/get_attribute_spec.rb +7 -0
- data/spec/actions/get_contents.rb +7 -0
- data/spec/actions/get_text.rb +7 -0
- data/spec/actions/prepend_with_spec.rb +30 -0
- data/spec/actions/remove_after.rb +7 -0
- data/spec/actions/remove_before.rb +7 -0
- data/spec/actions/replace_spec.rb +7 -0
- data/spec/actions/run_recipe_and_save_spec.tmp.rb +52 -0
- data/spec/actions/run_recipe_spec.tmp.rb +53 -0
- data/spec/actions/save_spec.rb +7 -0
- data/spec/actions/select_matching_spec.rb +7 -0
- data/spec/actions/set_input_spec.rb +7 -0
- data/spec/actions/skip_tags_spec.rb +7 -0
- data/spec/actions/split_spec.rb +7 -0
- data/spec/core/action_spec.rb +151 -0
- data/spec/core/argument_spec.rb +79 -0
- data/spec/core/browser_spec.rb +7 -0
- data/spec/core/dsl_spec.rb +7 -0
- data/spec/core/item_spec.rb +7 -0
- data/spec/core/keychain_spec.rb +7 -0
- data/spec/core/logger_spec.rb +7 -0
- data/spec/core/namespace_spec.rb +18 -0
- data/spec/core/recipe_spec.rb +81 -0
- data/spec/core/runtime_spec.rb +165 -0
- data/spec/core/type_spec.rb +7 -0
- data/spec/items/atom_spec.rb +7 -0
- data/spec/items/document_spec.rb +7 -0
- data/spec/items/html_spec.rb +7 -0
- data/spec/items/json_spec.rb +7 -0
- data/spec/items/markdown_spec.rb +7 -0
- data/spec/items/nothing_spec.rb +7 -0
- data/spec/items/text_spec.rb +17 -0
- data/spec/items/time_spec.rb +7 -0
- data/spec/items/url_spec.rb +7 -0
- data/spec/items/xml_spec.rb +17 -0
- data/spec/spec_helper.rb +22 -0
- metadata +355 -0
@@ -0,0 +1,18 @@
|
|
1
|
+
module Saper
|
2
|
+
module Actions
|
3
|
+
class Find < Action
|
4
|
+
|
5
|
+
returns_multiple_items!
|
6
|
+
argument :xpath
|
7
|
+
|
8
|
+
accepts :html, :returns => :html
|
9
|
+
accepts :xml, :returns => :xml
|
10
|
+
accepts :json, :returns => :json
|
11
|
+
|
12
|
+
run do |input, xpath|
|
13
|
+
input.find_all(xpath)
|
14
|
+
end
|
15
|
+
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
module Saper
|
2
|
+
module Actions
|
3
|
+
class FindFirst < Action
|
4
|
+
|
5
|
+
argument :xpath
|
6
|
+
accepts :html, :returns => :html
|
7
|
+
accepts :xml, :returns => :xml
|
8
|
+
accepts :json, :returns => :json
|
9
|
+
|
10
|
+
run do |input, xpath|
|
11
|
+
input.find(xpath)
|
12
|
+
end
|
13
|
+
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
module Saper
|
2
|
+
module Actions
|
3
|
+
class RunRecipe < Action
|
4
|
+
|
5
|
+
argument :recipe
|
6
|
+
accepts :anything, :returns => Proc.new { |instance| nil } # TODO
|
7
|
+
|
8
|
+
run do |input, subrecipe|
|
9
|
+
recipe(subrecipe, input)
|
10
|
+
end
|
11
|
+
|
12
|
+
# Overrides default logic
|
13
|
+
def multiple?
|
14
|
+
args.first.multiple?
|
15
|
+
end
|
16
|
+
|
17
|
+
# Overrides default logic
|
18
|
+
def requires
|
19
|
+
args.first.input_required
|
20
|
+
end
|
21
|
+
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
module Saper
|
2
|
+
module Actions
|
3
|
+
class RunRecipeAndSave < Action
|
4
|
+
|
5
|
+
argument :variable
|
6
|
+
argument :recipe
|
7
|
+
accepts :anything
|
8
|
+
|
9
|
+
run do |input, variable, subrecipe|
|
10
|
+
result = recipe(subrecipe, input)
|
11
|
+
self[variable] = result unless result.nil?
|
12
|
+
input
|
13
|
+
end
|
14
|
+
|
15
|
+
# Overrides default logic
|
16
|
+
def requires
|
17
|
+
args.last.input_required
|
18
|
+
end
|
19
|
+
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
module Saper
|
2
|
+
module Actions
|
3
|
+
class SkipTags < Action
|
4
|
+
|
5
|
+
argument :xpath
|
6
|
+
accepts :xml, :returns => :xml
|
7
|
+
accepts :html, :returns => :html
|
8
|
+
|
9
|
+
run do |input, xpath|
|
10
|
+
input.remove_children_preserving_content(xpath)
|
11
|
+
end
|
12
|
+
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
module Saper
|
2
|
+
module Actions
|
3
|
+
class Split < Action
|
4
|
+
|
5
|
+
returns_multiple_items!
|
6
|
+
argument :text
|
7
|
+
accepts :text, :returns => :text
|
8
|
+
accepts :document, :returns => :text
|
9
|
+
accepts :html, :returns => :html
|
10
|
+
|
11
|
+
run do |input, separator|
|
12
|
+
case input
|
13
|
+
when Items::HTML
|
14
|
+
input.inner_html.split(separator).map { |string| Item.new(:text, string).to_html }
|
15
|
+
when Items::Text
|
16
|
+
input.split(separator)
|
17
|
+
when Items::Document
|
18
|
+
input.to_text.split(separator)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
module Saper
|
2
|
+
module Arguments
|
3
|
+
class Recipe < Argument
|
4
|
+
|
5
|
+
def valid?(value)
|
6
|
+
value.is_a?(Symbol) || value.is_a?(String)
|
7
|
+
end
|
8
|
+
|
9
|
+
def normalize(value)
|
10
|
+
if value.is_a?(Recipe)
|
11
|
+
return value
|
12
|
+
end
|
13
|
+
if value.is_a?(String)
|
14
|
+
value = value.to_sym
|
15
|
+
end
|
16
|
+
unless value.is_a?(Symbol)
|
17
|
+
raise InvalidArgument
|
18
|
+
end
|
19
|
+
if action.nil?
|
20
|
+
raise NamespaceMissing
|
21
|
+
end
|
22
|
+
if action.namespace.nil?
|
23
|
+
raise NamespaceMissing
|
24
|
+
end
|
25
|
+
unless value = action.namespace[value]
|
26
|
+
raise ActionNotFound
|
27
|
+
end
|
28
|
+
value
|
29
|
+
end
|
30
|
+
|
31
|
+
#
|
32
|
+
def serialize
|
33
|
+
value.id
|
34
|
+
end
|
35
|
+
|
36
|
+
def to_string
|
37
|
+
value.id.inspect
|
38
|
+
end
|
39
|
+
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|