saper 0.5.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/LICENSE +21 -0
- data/README.md +126 -0
- data/Rakefile +17 -0
- data/bin/saper +60 -0
- data/lib/lib/json_search.rb +54 -0
- data/lib/lib/mechanize.rb +26 -0
- data/lib/lib/nokogiri.rb +12 -0
- data/lib/saper.rb +37 -0
- data/lib/saper/actions/append_with.rb +14 -0
- data/lib/saper/actions/convert_to_html.rb +14 -0
- data/lib/saper/actions/convert_to_json.rb +14 -0
- data/lib/saper/actions/convert_to_markdown.rb +13 -0
- data/lib/saper/actions/convert_to_time.rb +15 -0
- data/lib/saper/actions/convert_to_xml.rb +14 -0
- data/lib/saper/actions/create_atom.rb +18 -0
- data/lib/saper/actions/fetch.rb +17 -0
- data/lib/saper/actions/find.rb +18 -0
- data/lib/saper/actions/find_first.rb +16 -0
- data/lib/saper/actions/get_attribute.rb +15 -0
- data/lib/saper/actions/get_contents.rb +14 -0
- data/lib/saper/actions/get_text.rb +14 -0
- data/lib/saper/actions/prepend_with.rb +14 -0
- data/lib/saper/actions/remove_after.rb +14 -0
- data/lib/saper/actions/remove_before.rb +14 -0
- data/lib/saper/actions/remove_matching.rb +14 -0
- data/lib/saper/actions/remove_tags.rb +15 -0
- data/lib/saper/actions/replace.rb +15 -0
- data/lib/saper/actions/run_recipe.rb +24 -0
- data/lib/saper/actions/run_recipe_and_save.rb +22 -0
- data/lib/saper/actions/save.rb +14 -0
- data/lib/saper/actions/select_matching.rb +14 -0
- data/lib/saper/actions/set_input.rb +19 -0
- data/lib/saper/actions/skip_tags.rb +15 -0
- data/lib/saper/actions/split.rb +24 -0
- data/lib/saper/arguments/attribute.rb +11 -0
- data/lib/saper/arguments/recipe.rb +42 -0
- data/lib/saper/arguments/text.rb +11 -0
- data/lib/saper/arguments/timezone.rb +11 -0
- data/lib/saper/arguments/variable.rb +11 -0
- data/lib/saper/arguments/xpath.rb +11 -0
- data/lib/saper/core/action.rb +209 -0
- data/lib/saper/core/argument.rb +106 -0
- data/lib/saper/core/browser.rb +87 -0
- data/lib/saper/core/dsl.rb +68 -0
- data/lib/saper/core/error.rb +47 -0
- data/lib/saper/core/item.rb +70 -0
- data/lib/saper/core/keychain.rb +18 -0
- data/lib/saper/core/logger.rb +74 -0
- data/lib/saper/core/namespace.rb +139 -0
- data/lib/saper/core/recipe.rb +134 -0
- data/lib/saper/core/runtime.rb +237 -0
- data/lib/saper/core/type.rb +45 -0
- data/lib/saper/items/atom.rb +64 -0
- data/lib/saper/items/document.rb +66 -0
- data/lib/saper/items/html.rb +85 -0
- data/lib/saper/items/json.rb +67 -0
- data/lib/saper/items/markdown.rb +36 -0
- data/lib/saper/items/nothing.rb +15 -0
- data/lib/saper/items/text.rb +54 -0
- data/lib/saper/items/time.rb +42 -0
- data/lib/saper/items/url.rb +34 -0
- data/lib/saper/items/xml.rb +79 -0
- data/lib/saper/version.rb +3 -0
- data/spec/actions/append_with_spec.rb +30 -0
- data/spec/actions/convert_to_html_spec.rb +24 -0
- data/spec/actions/convert_to_json_spec.rb +24 -0
- data/spec/actions/convert_to_markdown_spec.rb +24 -0
- data/spec/actions/convert_to_time_spec.rb +37 -0
- data/spec/actions/convert_to_xml_spec.rb +24 -0
- data/spec/actions/create_atom_spec.rb +31 -0
- data/spec/actions/fetch_spec.rb +7 -0
- data/spec/actions/find_first_spec.rb +7 -0
- data/spec/actions/find_spec.rb +7 -0
- data/spec/actions/get_attribute_spec.rb +7 -0
- data/spec/actions/get_contents.rb +7 -0
- data/spec/actions/get_text.rb +7 -0
- data/spec/actions/prepend_with_spec.rb +30 -0
- data/spec/actions/remove_after.rb +7 -0
- data/spec/actions/remove_before.rb +7 -0
- data/spec/actions/replace_spec.rb +7 -0
- data/spec/actions/run_recipe_and_save_spec.tmp.rb +52 -0
- data/spec/actions/run_recipe_spec.tmp.rb +53 -0
- data/spec/actions/save_spec.rb +7 -0
- data/spec/actions/select_matching_spec.rb +7 -0
- data/spec/actions/set_input_spec.rb +7 -0
- data/spec/actions/skip_tags_spec.rb +7 -0
- data/spec/actions/split_spec.rb +7 -0
- data/spec/core/action_spec.rb +151 -0
- data/spec/core/argument_spec.rb +79 -0
- data/spec/core/browser_spec.rb +7 -0
- data/spec/core/dsl_spec.rb +7 -0
- data/spec/core/item_spec.rb +7 -0
- data/spec/core/keychain_spec.rb +7 -0
- data/spec/core/logger_spec.rb +7 -0
- data/spec/core/namespace_spec.rb +18 -0
- data/spec/core/recipe_spec.rb +81 -0
- data/spec/core/runtime_spec.rb +165 -0
- data/spec/core/type_spec.rb +7 -0
- data/spec/items/atom_spec.rb +7 -0
- data/spec/items/document_spec.rb +7 -0
- data/spec/items/html_spec.rb +7 -0
- data/spec/items/json_spec.rb +7 -0
- data/spec/items/markdown_spec.rb +7 -0
- data/spec/items/nothing_spec.rb +7 -0
- data/spec/items/text_spec.rb +17 -0
- data/spec/items/time_spec.rb +7 -0
- data/spec/items/url_spec.rb +7 -0
- data/spec/items/xml_spec.rb +17 -0
- data/spec/spec_helper.rb +22 -0
- metadata +355 -0
@@ -0,0 +1,67 @@
|
|
1
|
+
module Saper
|
2
|
+
module Items
|
3
|
+
class JSON < Item
|
4
|
+
|
5
|
+
def self.new(item)
|
6
|
+
super case item
|
7
|
+
when Text
|
8
|
+
parse(item.to_s)
|
9
|
+
when String
|
10
|
+
parse(item)
|
11
|
+
else
|
12
|
+
raise(InvalidItem, item)
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.parse(string)
|
17
|
+
begin
|
18
|
+
::JSON.parse(string)
|
19
|
+
rescue
|
20
|
+
raise(InvalidItem, string)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
def initialize(hash)
|
25
|
+
@hash = hash
|
26
|
+
end
|
27
|
+
|
28
|
+
def [](name)
|
29
|
+
value = @hash[name.to_s]
|
30
|
+
case value
|
31
|
+
when Hash
|
32
|
+
self.class.new(value)
|
33
|
+
when Array
|
34
|
+
self.class.new(value)
|
35
|
+
else
|
36
|
+
value
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
def find(xpath)
|
41
|
+
find_all(xpath).first
|
42
|
+
end
|
43
|
+
|
44
|
+
def find_all(xpath)
|
45
|
+
JSONSearch.find(@hash, xpath).map do |item|
|
46
|
+
sanitize(item)
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
def sanitize(item)
|
51
|
+
case item
|
52
|
+
when Hash
|
53
|
+
self.class.new(item)
|
54
|
+
when Array
|
55
|
+
item.map { |i| sanitize(i) }
|
56
|
+
else
|
57
|
+
item
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
def to_native
|
62
|
+
@hash
|
63
|
+
end
|
64
|
+
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
module Saper
|
2
|
+
module Items
|
3
|
+
class Markdown < Item
|
4
|
+
|
5
|
+
def self.new(item)
|
6
|
+
super case item
|
7
|
+
when HTML
|
8
|
+
parse(item.inner_html)
|
9
|
+
else
|
10
|
+
raise(InvalidItem, item)
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
def self.parse(string)
|
15
|
+
begin
|
16
|
+
::ReverseMarkdown.convert(string).strip
|
17
|
+
rescue
|
18
|
+
raise(InvalidItem, string)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
def initialize(string)
|
23
|
+
@string = string
|
24
|
+
end
|
25
|
+
|
26
|
+
def to_s
|
27
|
+
@string
|
28
|
+
end
|
29
|
+
|
30
|
+
def to_native
|
31
|
+
to_s
|
32
|
+
end
|
33
|
+
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
@@ -0,0 +1,54 @@
|
|
1
|
+
module Saper
|
2
|
+
module Items
|
3
|
+
class Text < Item
|
4
|
+
|
5
|
+
def self.new(item)
|
6
|
+
super case item
|
7
|
+
when String
|
8
|
+
item
|
9
|
+
when Text
|
10
|
+
item.to_s
|
11
|
+
else
|
12
|
+
raise(InvalidItem, item)
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
def initialize(string)
|
17
|
+
@string = string
|
18
|
+
end
|
19
|
+
|
20
|
+
def split(separator, number = nil)
|
21
|
+
@string.split(separator, number)
|
22
|
+
end
|
23
|
+
|
24
|
+
def gsub(*args, &block)
|
25
|
+
@string.gsub(*args, &block)
|
26
|
+
end
|
27
|
+
|
28
|
+
def to_xml
|
29
|
+
XML.new(self)
|
30
|
+
end
|
31
|
+
|
32
|
+
def to_html
|
33
|
+
HTML.new(self)
|
34
|
+
end
|
35
|
+
|
36
|
+
def to_json
|
37
|
+
JSON.new(self)
|
38
|
+
end
|
39
|
+
|
40
|
+
def to_time(format, tz = nil)
|
41
|
+
Time.new(self, format, tz)
|
42
|
+
end
|
43
|
+
|
44
|
+
def to_s
|
45
|
+
@string
|
46
|
+
end
|
47
|
+
|
48
|
+
def to_native
|
49
|
+
to_s
|
50
|
+
end
|
51
|
+
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
module Saper
|
2
|
+
module Items
|
3
|
+
class Time < Item
|
4
|
+
|
5
|
+
def self.new(item, format, tz = nil)
|
6
|
+
super case item
|
7
|
+
when Text
|
8
|
+
parse(item.to_s, format, tz)
|
9
|
+
when String
|
10
|
+
parse(item, format, tz)
|
11
|
+
else
|
12
|
+
raise(InvalidItem, item)
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.parse(string, format, tz = nil)
|
17
|
+
begin
|
18
|
+
::Vremya.parse(string, format, tz)
|
19
|
+
rescue
|
20
|
+
raise(InvalidItem, string)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
def initialize(vremya)
|
25
|
+
@vremya = vremya
|
26
|
+
end
|
27
|
+
|
28
|
+
def to_s
|
29
|
+
@vremya.iso8601
|
30
|
+
end
|
31
|
+
|
32
|
+
def to_native
|
33
|
+
@vremya.to_time
|
34
|
+
end
|
35
|
+
|
36
|
+
def to_json(*args)
|
37
|
+
@vremya.to_i.to_s
|
38
|
+
end
|
39
|
+
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
module Saper
|
2
|
+
module Items
|
3
|
+
class URL < Item
|
4
|
+
|
5
|
+
def self.new(item)
|
6
|
+
item = case item
|
7
|
+
when Text
|
8
|
+
item.to_s
|
9
|
+
when String
|
10
|
+
item
|
11
|
+
else
|
12
|
+
raise(InvalidItem, item)
|
13
|
+
end
|
14
|
+
unless item =~ /^#{URI::regexp}$/
|
15
|
+
raise(InvalidItem, item)
|
16
|
+
end
|
17
|
+
super(item)
|
18
|
+
end
|
19
|
+
|
20
|
+
def initialize(url)
|
21
|
+
@url = url
|
22
|
+
end
|
23
|
+
|
24
|
+
def to_s
|
25
|
+
@url
|
26
|
+
end
|
27
|
+
|
28
|
+
def to_native
|
29
|
+
to_s
|
30
|
+
end
|
31
|
+
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
@@ -0,0 +1,79 @@
|
|
1
|
+
module Saper
|
2
|
+
module Items
|
3
|
+
class XML < Item
|
4
|
+
|
5
|
+
def self.new(item)
|
6
|
+
super case item
|
7
|
+
when Nokogiri::XML
|
8
|
+
item
|
9
|
+
when Text
|
10
|
+
parse(item.to_s)
|
11
|
+
when String
|
12
|
+
parse(item)
|
13
|
+
else
|
14
|
+
raise(InvalidItem, item)
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def self.parse(string, uri = nil, charset = nil)
|
19
|
+
begin
|
20
|
+
Nokogiri::XML.parse(string, uri, charset)
|
21
|
+
rescue
|
22
|
+
raise(InvalidItem, string)
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def initialize(noko)
|
27
|
+
@noko = noko
|
28
|
+
# Force UTF-8 encoding
|
29
|
+
# https://github.com/sparklemotion/nokogiri/issues/117
|
30
|
+
@noko.document.encoding = 'UTF-8'
|
31
|
+
end
|
32
|
+
|
33
|
+
def name
|
34
|
+
@noko.name
|
35
|
+
end
|
36
|
+
|
37
|
+
def [](name)
|
38
|
+
@noko[name]
|
39
|
+
end
|
40
|
+
|
41
|
+
def find(xpath)
|
42
|
+
find_all(xpath).first
|
43
|
+
end
|
44
|
+
|
45
|
+
def find_all(xpath)
|
46
|
+
@noko.search(xpath).map { |element| XML.new(element) }
|
47
|
+
end
|
48
|
+
|
49
|
+
def remove_children_with_content(xpath)
|
50
|
+
@noko.search(xpath).each { |item| item.remove }; self
|
51
|
+
end
|
52
|
+
|
53
|
+
def remove_children_preserving_content(xpath)
|
54
|
+
@noko.search(xpath).each { |item| item.replace(item.children) }; self
|
55
|
+
end
|
56
|
+
|
57
|
+
def remove(tag)
|
58
|
+
remove_children_preserving_content(tag)
|
59
|
+
end
|
60
|
+
|
61
|
+
def inner_html
|
62
|
+
@noko.inner_html
|
63
|
+
end
|
64
|
+
|
65
|
+
def inner_text
|
66
|
+
@noko.inner_text
|
67
|
+
end
|
68
|
+
|
69
|
+
def to_s
|
70
|
+
inner_html
|
71
|
+
end
|
72
|
+
|
73
|
+
def to_native
|
74
|
+
inner_html
|
75
|
+
end
|
76
|
+
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
require './spec/spec_helper'
|
2
|
+
|
3
|
+
describe Saper::Actions::AppendWith do
|
4
|
+
|
5
|
+
let(:action) do
|
6
|
+
Saper::Action.new(:append_with, '!')
|
7
|
+
end
|
8
|
+
|
9
|
+
context ".accepts" do
|
10
|
+
it "returns acceptable input types" do
|
11
|
+
Saper::Actions::AppendWith.accepts.should == [:text]
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
context ".new" do
|
16
|
+
it "raises InvalidArgument when argument is missing" do
|
17
|
+
expect { subject }.to raise_error(Saper::InvalidArgument)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
context "#run" do
|
22
|
+
it "raises InvalidInput when input is nil" do
|
23
|
+
expect { action.run(nil) }.to raise_error(Saper::InvalidInput)
|
24
|
+
end
|
25
|
+
it "returns valid output when input is Text" do
|
26
|
+
action.run('string').should == 'string!'
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
require './spec/spec_helper'
|
2
|
+
|
3
|
+
describe Saper::Actions::ConvertToHTML do
|
4
|
+
|
5
|
+
let(:action) do
|
6
|
+
Saper::Action.new(:convert_to_html)
|
7
|
+
end
|
8
|
+
|
9
|
+
context ".accepts" do
|
10
|
+
it "returns acceptable input types" do
|
11
|
+
Saper::Actions::ConvertToHTML.accepts.should == [:text, :document]
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
context "#run" do
|
16
|
+
it "raises InvalidInput when input is nil" do
|
17
|
+
expect { action.run(nil) }.to raise_error(Saper::InvalidInput)
|
18
|
+
end
|
19
|
+
it "raises no errors when input is a valid Text" do
|
20
|
+
expect { action.run('<p>test</p>') }.to_not raise_error
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
require './spec/spec_helper'
|
2
|
+
|
3
|
+
describe Saper::Actions::ConvertToJSON do
|
4
|
+
|
5
|
+
let(:action) do
|
6
|
+
Saper::Action.new(:convert_to_json)
|
7
|
+
end
|
8
|
+
|
9
|
+
context ".accepts" do
|
10
|
+
it "returns acceptable input types" do
|
11
|
+
Saper::Actions::ConvertToJSON.accepts.should == [:text, :document]
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
context "#run" do
|
16
|
+
it "raises InvalidInput when input is nil" do
|
17
|
+
expect { action.run(nil) }.to raise_error(Saper::InvalidInput)
|
18
|
+
end
|
19
|
+
it "raises no errors when input is a valid Text" do
|
20
|
+
expect { action.run('{"test":"ok"}') }.to_not raise_error
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
require './spec/spec_helper'
|
2
|
+
|
3
|
+
describe Saper::Actions::ConvertToMarkdown do
|
4
|
+
|
5
|
+
let(:action) do
|
6
|
+
Saper::Action.new(:convert_to_markdown)
|
7
|
+
end
|
8
|
+
|
9
|
+
context ".accepts" do
|
10
|
+
it "returns acceptable input types" do
|
11
|
+
Saper::Actions::ConvertToMarkdown.accepts.should == [:html]
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
context "#run" do
|
16
|
+
it "raises InvalidInput when input is nil" do
|
17
|
+
expect { action.run(nil) }.to raise_error(Saper::InvalidInput)
|
18
|
+
end
|
19
|
+
it "returns valid markdown when input is HTML" do
|
20
|
+
action.run(Saper::Items::HTML.new('<p>test</p>')).should == 'test'
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
end
|