saper 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +21 -0
- data/README.md +126 -0
- data/Rakefile +17 -0
- data/bin/saper +60 -0
- data/lib/lib/json_search.rb +54 -0
- data/lib/lib/mechanize.rb +26 -0
- data/lib/lib/nokogiri.rb +12 -0
- data/lib/saper.rb +37 -0
- data/lib/saper/actions/append_with.rb +14 -0
- data/lib/saper/actions/convert_to_html.rb +14 -0
- data/lib/saper/actions/convert_to_json.rb +14 -0
- data/lib/saper/actions/convert_to_markdown.rb +13 -0
- data/lib/saper/actions/convert_to_time.rb +15 -0
- data/lib/saper/actions/convert_to_xml.rb +14 -0
- data/lib/saper/actions/create_atom.rb +18 -0
- data/lib/saper/actions/fetch.rb +17 -0
- data/lib/saper/actions/find.rb +18 -0
- data/lib/saper/actions/find_first.rb +16 -0
- data/lib/saper/actions/get_attribute.rb +15 -0
- data/lib/saper/actions/get_contents.rb +14 -0
- data/lib/saper/actions/get_text.rb +14 -0
- data/lib/saper/actions/prepend_with.rb +14 -0
- data/lib/saper/actions/remove_after.rb +14 -0
- data/lib/saper/actions/remove_before.rb +14 -0
- data/lib/saper/actions/remove_matching.rb +14 -0
- data/lib/saper/actions/remove_tags.rb +15 -0
- data/lib/saper/actions/replace.rb +15 -0
- data/lib/saper/actions/run_recipe.rb +24 -0
- data/lib/saper/actions/run_recipe_and_save.rb +22 -0
- data/lib/saper/actions/save.rb +14 -0
- data/lib/saper/actions/select_matching.rb +14 -0
- data/lib/saper/actions/set_input.rb +19 -0
- data/lib/saper/actions/skip_tags.rb +15 -0
- data/lib/saper/actions/split.rb +24 -0
- data/lib/saper/arguments/attribute.rb +11 -0
- data/lib/saper/arguments/recipe.rb +42 -0
- data/lib/saper/arguments/text.rb +11 -0
- data/lib/saper/arguments/timezone.rb +11 -0
- data/lib/saper/arguments/variable.rb +11 -0
- data/lib/saper/arguments/xpath.rb +11 -0
- data/lib/saper/core/action.rb +209 -0
- data/lib/saper/core/argument.rb +106 -0
- data/lib/saper/core/browser.rb +87 -0
- data/lib/saper/core/dsl.rb +68 -0
- data/lib/saper/core/error.rb +47 -0
- data/lib/saper/core/item.rb +70 -0
- data/lib/saper/core/keychain.rb +18 -0
- data/lib/saper/core/logger.rb +74 -0
- data/lib/saper/core/namespace.rb +139 -0
- data/lib/saper/core/recipe.rb +134 -0
- data/lib/saper/core/runtime.rb +237 -0
- data/lib/saper/core/type.rb +45 -0
- data/lib/saper/items/atom.rb +64 -0
- data/lib/saper/items/document.rb +66 -0
- data/lib/saper/items/html.rb +85 -0
- data/lib/saper/items/json.rb +67 -0
- data/lib/saper/items/markdown.rb +36 -0
- data/lib/saper/items/nothing.rb +15 -0
- data/lib/saper/items/text.rb +54 -0
- data/lib/saper/items/time.rb +42 -0
- data/lib/saper/items/url.rb +34 -0
- data/lib/saper/items/xml.rb +79 -0
- data/lib/saper/version.rb +3 -0
- data/spec/actions/append_with_spec.rb +30 -0
- data/spec/actions/convert_to_html_spec.rb +24 -0
- data/spec/actions/convert_to_json_spec.rb +24 -0
- data/spec/actions/convert_to_markdown_spec.rb +24 -0
- data/spec/actions/convert_to_time_spec.rb +37 -0
- data/spec/actions/convert_to_xml_spec.rb +24 -0
- data/spec/actions/create_atom_spec.rb +31 -0
- data/spec/actions/fetch_spec.rb +7 -0
- data/spec/actions/find_first_spec.rb +7 -0
- data/spec/actions/find_spec.rb +7 -0
- data/spec/actions/get_attribute_spec.rb +7 -0
- data/spec/actions/get_contents.rb +7 -0
- data/spec/actions/get_text.rb +7 -0
- data/spec/actions/prepend_with_spec.rb +30 -0
- data/spec/actions/remove_after.rb +7 -0
- data/spec/actions/remove_before.rb +7 -0
- data/spec/actions/replace_spec.rb +7 -0
- data/spec/actions/run_recipe_and_save_spec.tmp.rb +52 -0
- data/spec/actions/run_recipe_spec.tmp.rb +53 -0
- data/spec/actions/save_spec.rb +7 -0
- data/spec/actions/select_matching_spec.rb +7 -0
- data/spec/actions/set_input_spec.rb +7 -0
- data/spec/actions/skip_tags_spec.rb +7 -0
- data/spec/actions/split_spec.rb +7 -0
- data/spec/core/action_spec.rb +151 -0
- data/spec/core/argument_spec.rb +79 -0
- data/spec/core/browser_spec.rb +7 -0
- data/spec/core/dsl_spec.rb +7 -0
- data/spec/core/item_spec.rb +7 -0
- data/spec/core/keychain_spec.rb +7 -0
- data/spec/core/logger_spec.rb +7 -0
- data/spec/core/namespace_spec.rb +18 -0
- data/spec/core/recipe_spec.rb +81 -0
- data/spec/core/runtime_spec.rb +165 -0
- data/spec/core/type_spec.rb +7 -0
- data/spec/items/atom_spec.rb +7 -0
- data/spec/items/document_spec.rb +7 -0
- data/spec/items/html_spec.rb +7 -0
- data/spec/items/json_spec.rb +7 -0
- data/spec/items/markdown_spec.rb +7 -0
- data/spec/items/nothing_spec.rb +7 -0
- data/spec/items/text_spec.rb +17 -0
- data/spec/items/time_spec.rb +7 -0
- data/spec/items/url_spec.rb +7 -0
- data/spec/items/xml_spec.rb +17 -0
- data/spec/spec_helper.rb +22 -0
- metadata +355 -0
@@ -0,0 +1,67 @@
|
|
1
|
+
module Saper
|
2
|
+
module Items
|
3
|
+
class JSON < Item
|
4
|
+
|
5
|
+
def self.new(item)
|
6
|
+
super case item
|
7
|
+
when Text
|
8
|
+
parse(item.to_s)
|
9
|
+
when String
|
10
|
+
parse(item)
|
11
|
+
else
|
12
|
+
raise(InvalidItem, item)
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.parse(string)
|
17
|
+
begin
|
18
|
+
::JSON.parse(string)
|
19
|
+
rescue
|
20
|
+
raise(InvalidItem, string)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
def initialize(hash)
|
25
|
+
@hash = hash
|
26
|
+
end
|
27
|
+
|
28
|
+
def [](name)
|
29
|
+
value = @hash[name.to_s]
|
30
|
+
case value
|
31
|
+
when Hash
|
32
|
+
self.class.new(value)
|
33
|
+
when Array
|
34
|
+
self.class.new(value)
|
35
|
+
else
|
36
|
+
value
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
def find(xpath)
|
41
|
+
find_all(xpath).first
|
42
|
+
end
|
43
|
+
|
44
|
+
def find_all(xpath)
|
45
|
+
JSONSearch.find(@hash, xpath).map do |item|
|
46
|
+
sanitize(item)
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
def sanitize(item)
|
51
|
+
case item
|
52
|
+
when Hash
|
53
|
+
self.class.new(item)
|
54
|
+
when Array
|
55
|
+
item.map { |i| sanitize(i) }
|
56
|
+
else
|
57
|
+
item
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
def to_native
|
62
|
+
@hash
|
63
|
+
end
|
64
|
+
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
module Saper
|
2
|
+
module Items
|
3
|
+
class Markdown < Item
|
4
|
+
|
5
|
+
def self.new(item)
|
6
|
+
super case item
|
7
|
+
when HTML
|
8
|
+
parse(item.inner_html)
|
9
|
+
else
|
10
|
+
raise(InvalidItem, item)
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
def self.parse(string)
|
15
|
+
begin
|
16
|
+
::ReverseMarkdown.convert(string).strip
|
17
|
+
rescue
|
18
|
+
raise(InvalidItem, string)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
def initialize(string)
|
23
|
+
@string = string
|
24
|
+
end
|
25
|
+
|
26
|
+
def to_s
|
27
|
+
@string
|
28
|
+
end
|
29
|
+
|
30
|
+
def to_native
|
31
|
+
to_s
|
32
|
+
end
|
33
|
+
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
@@ -0,0 +1,54 @@
|
|
1
|
+
module Saper
|
2
|
+
module Items
|
3
|
+
class Text < Item
|
4
|
+
|
5
|
+
def self.new(item)
|
6
|
+
super case item
|
7
|
+
when String
|
8
|
+
item
|
9
|
+
when Text
|
10
|
+
item.to_s
|
11
|
+
else
|
12
|
+
raise(InvalidItem, item)
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
def initialize(string)
|
17
|
+
@string = string
|
18
|
+
end
|
19
|
+
|
20
|
+
def split(separator, number = nil)
|
21
|
+
@string.split(separator, number)
|
22
|
+
end
|
23
|
+
|
24
|
+
def gsub(*args, &block)
|
25
|
+
@string.gsub(*args, &block)
|
26
|
+
end
|
27
|
+
|
28
|
+
def to_xml
|
29
|
+
XML.new(self)
|
30
|
+
end
|
31
|
+
|
32
|
+
def to_html
|
33
|
+
HTML.new(self)
|
34
|
+
end
|
35
|
+
|
36
|
+
def to_json
|
37
|
+
JSON.new(self)
|
38
|
+
end
|
39
|
+
|
40
|
+
def to_time(format, tz = nil)
|
41
|
+
Time.new(self, format, tz)
|
42
|
+
end
|
43
|
+
|
44
|
+
def to_s
|
45
|
+
@string
|
46
|
+
end
|
47
|
+
|
48
|
+
def to_native
|
49
|
+
to_s
|
50
|
+
end
|
51
|
+
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
module Saper
|
2
|
+
module Items
|
3
|
+
class Time < Item
|
4
|
+
|
5
|
+
def self.new(item, format, tz = nil)
|
6
|
+
super case item
|
7
|
+
when Text
|
8
|
+
parse(item.to_s, format, tz)
|
9
|
+
when String
|
10
|
+
parse(item, format, tz)
|
11
|
+
else
|
12
|
+
raise(InvalidItem, item)
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.parse(string, format, tz = nil)
|
17
|
+
begin
|
18
|
+
::Vremya.parse(string, format, tz)
|
19
|
+
rescue
|
20
|
+
raise(InvalidItem, string)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
def initialize(vremya)
|
25
|
+
@vremya = vremya
|
26
|
+
end
|
27
|
+
|
28
|
+
def to_s
|
29
|
+
@vremya.iso8601
|
30
|
+
end
|
31
|
+
|
32
|
+
def to_native
|
33
|
+
@vremya.to_time
|
34
|
+
end
|
35
|
+
|
36
|
+
def to_json(*args)
|
37
|
+
@vremya.to_i.to_s
|
38
|
+
end
|
39
|
+
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
module Saper
|
2
|
+
module Items
|
3
|
+
class URL < Item
|
4
|
+
|
5
|
+
def self.new(item)
|
6
|
+
item = case item
|
7
|
+
when Text
|
8
|
+
item.to_s
|
9
|
+
when String
|
10
|
+
item
|
11
|
+
else
|
12
|
+
raise(InvalidItem, item)
|
13
|
+
end
|
14
|
+
unless item =~ /^#{URI::regexp}$/
|
15
|
+
raise(InvalidItem, item)
|
16
|
+
end
|
17
|
+
super(item)
|
18
|
+
end
|
19
|
+
|
20
|
+
def initialize(url)
|
21
|
+
@url = url
|
22
|
+
end
|
23
|
+
|
24
|
+
def to_s
|
25
|
+
@url
|
26
|
+
end
|
27
|
+
|
28
|
+
def to_native
|
29
|
+
to_s
|
30
|
+
end
|
31
|
+
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
@@ -0,0 +1,79 @@
|
|
1
|
+
module Saper
|
2
|
+
module Items
|
3
|
+
class XML < Item
|
4
|
+
|
5
|
+
def self.new(item)
|
6
|
+
super case item
|
7
|
+
when Nokogiri::XML
|
8
|
+
item
|
9
|
+
when Text
|
10
|
+
parse(item.to_s)
|
11
|
+
when String
|
12
|
+
parse(item)
|
13
|
+
else
|
14
|
+
raise(InvalidItem, item)
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def self.parse(string, uri = nil, charset = nil)
|
19
|
+
begin
|
20
|
+
Nokogiri::XML.parse(string, uri, charset)
|
21
|
+
rescue
|
22
|
+
raise(InvalidItem, string)
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def initialize(noko)
|
27
|
+
@noko = noko
|
28
|
+
# Force UTF-8 encoding
|
29
|
+
# https://github.com/sparklemotion/nokogiri/issues/117
|
30
|
+
@noko.document.encoding = 'UTF-8'
|
31
|
+
end
|
32
|
+
|
33
|
+
def name
|
34
|
+
@noko.name
|
35
|
+
end
|
36
|
+
|
37
|
+
def [](name)
|
38
|
+
@noko[name]
|
39
|
+
end
|
40
|
+
|
41
|
+
def find(xpath)
|
42
|
+
find_all(xpath).first
|
43
|
+
end
|
44
|
+
|
45
|
+
def find_all(xpath)
|
46
|
+
@noko.search(xpath).map { |element| XML.new(element) }
|
47
|
+
end
|
48
|
+
|
49
|
+
def remove_children_with_content(xpath)
|
50
|
+
@noko.search(xpath).each { |item| item.remove }; self
|
51
|
+
end
|
52
|
+
|
53
|
+
def remove_children_preserving_content(xpath)
|
54
|
+
@noko.search(xpath).each { |item| item.replace(item.children) }; self
|
55
|
+
end
|
56
|
+
|
57
|
+
def remove(tag)
|
58
|
+
remove_children_preserving_content(tag)
|
59
|
+
end
|
60
|
+
|
61
|
+
def inner_html
|
62
|
+
@noko.inner_html
|
63
|
+
end
|
64
|
+
|
65
|
+
def inner_text
|
66
|
+
@noko.inner_text
|
67
|
+
end
|
68
|
+
|
69
|
+
def to_s
|
70
|
+
inner_html
|
71
|
+
end
|
72
|
+
|
73
|
+
def to_native
|
74
|
+
inner_html
|
75
|
+
end
|
76
|
+
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
require './spec/spec_helper'
|
2
|
+
|
3
|
+
describe Saper::Actions::AppendWith do
|
4
|
+
|
5
|
+
let(:action) do
|
6
|
+
Saper::Action.new(:append_with, '!')
|
7
|
+
end
|
8
|
+
|
9
|
+
context ".accepts" do
|
10
|
+
it "returns acceptable input types" do
|
11
|
+
Saper::Actions::AppendWith.accepts.should == [:text]
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
context ".new" do
|
16
|
+
it "raises InvalidArgument when argument is missing" do
|
17
|
+
expect { subject }.to raise_error(Saper::InvalidArgument)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
context "#run" do
|
22
|
+
it "raises InvalidInput when input is nil" do
|
23
|
+
expect { action.run(nil) }.to raise_error(Saper::InvalidInput)
|
24
|
+
end
|
25
|
+
it "returns valid output when input is Text" do
|
26
|
+
action.run('string').should == 'string!'
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
require './spec/spec_helper'
|
2
|
+
|
3
|
+
describe Saper::Actions::ConvertToHTML do
|
4
|
+
|
5
|
+
let(:action) do
|
6
|
+
Saper::Action.new(:convert_to_html)
|
7
|
+
end
|
8
|
+
|
9
|
+
context ".accepts" do
|
10
|
+
it "returns acceptable input types" do
|
11
|
+
Saper::Actions::ConvertToHTML.accepts.should == [:text, :document]
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
context "#run" do
|
16
|
+
it "raises InvalidInput when input is nil" do
|
17
|
+
expect { action.run(nil) }.to raise_error(Saper::InvalidInput)
|
18
|
+
end
|
19
|
+
it "raises no errors when input is a valid Text" do
|
20
|
+
expect { action.run('<p>test</p>') }.to_not raise_error
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
require './spec/spec_helper'
|
2
|
+
|
3
|
+
describe Saper::Actions::ConvertToJSON do
|
4
|
+
|
5
|
+
let(:action) do
|
6
|
+
Saper::Action.new(:convert_to_json)
|
7
|
+
end
|
8
|
+
|
9
|
+
context ".accepts" do
|
10
|
+
it "returns acceptable input types" do
|
11
|
+
Saper::Actions::ConvertToJSON.accepts.should == [:text, :document]
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
context "#run" do
|
16
|
+
it "raises InvalidInput when input is nil" do
|
17
|
+
expect { action.run(nil) }.to raise_error(Saper::InvalidInput)
|
18
|
+
end
|
19
|
+
it "raises no errors when input is a valid Text" do
|
20
|
+
expect { action.run('{"test":"ok"}') }.to_not raise_error
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
require './spec/spec_helper'
|
2
|
+
|
3
|
+
describe Saper::Actions::ConvertToMarkdown do
|
4
|
+
|
5
|
+
let(:action) do
|
6
|
+
Saper::Action.new(:convert_to_markdown)
|
7
|
+
end
|
8
|
+
|
9
|
+
context ".accepts" do
|
10
|
+
it "returns acceptable input types" do
|
11
|
+
Saper::Actions::ConvertToMarkdown.accepts.should == [:html]
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
context "#run" do
|
16
|
+
it "raises InvalidInput when input is nil" do
|
17
|
+
expect { action.run(nil) }.to raise_error(Saper::InvalidInput)
|
18
|
+
end
|
19
|
+
it "returns valid markdown when input is HTML" do
|
20
|
+
action.run(Saper::Items::HTML.new('<p>test</p>')).should == 'test'
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
end
|