purehtml 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +7 -0
- data/README.md +173 -0
- data/Rakefile +12 -0
- data/lib/purehtml/backend.rb +120 -0
- data/lib/purehtml/config.rb +9 -0
- data/lib/purehtml/config_factory.rb +129 -0
- data/lib/purehtml/configs/array_config.rb +32 -0
- data/lib/purehtml/configs/constant_config.rb +26 -0
- data/lib/purehtml/configs/default_value_config.rb +22 -0
- data/lib/purehtml/configs/object_config.rb +29 -0
- data/lib/purehtml/configs/primitive_value_config.rb +36 -0
- data/lib/purehtml/configs/union_config.rb +47 -0
- data/lib/purehtml/configs/with_selector.rb +33 -0
- data/lib/purehtml/errors.rb +24 -0
- data/lib/purehtml/extract_params.rb +27 -0
- data/lib/purehtml/transformer.rb +19 -0
- data/lib/purehtml/transformer_factory.rb +76 -0
- data/lib/purehtml/transformers/attr.rb +28 -0
- data/lib/purehtml/transformers/boolean.rb +25 -0
- data/lib/purehtml/transformers/capitalize.rb +27 -0
- data/lib/purehtml/transformers/exists.rb +17 -0
- data/lib/purehtml/transformers/html.rb +17 -0
- data/lib/purehtml/transformers/join.rb +26 -0
- data/lib/purehtml/transformers/json.rb +22 -0
- data/lib/purehtml/transformers/jsonpath.rb +139 -0
- data/lib/purehtml/transformers/length.rb +23 -0
- data/lib/purehtml/transformers/lower.rb +25 -0
- data/lib/purehtml/transformers/number.rb +37 -0
- data/lib/purehtml/transformers/remove_last_path_section.rb +40 -0
- data/lib/purehtml/transformers/remove_url_query_param.rb +36 -0
- data/lib/purehtml/transformers/replace.rb +36 -0
- data/lib/purehtml/transformers/resolve.rb +24 -0
- data/lib/purehtml/transformers/split.rb +31 -0
- data/lib/purehtml/transformers/text.rb +17 -0
- data/lib/purehtml/transformers/trim.rb +23 -0
- data/lib/purehtml/transformers/upper.rb +25 -0
- data/lib/purehtml/transformers/url_query_param.rb +36 -0
- data/lib/purehtml/transformers.rb +53 -0
- data/lib/purehtml/version.rb +5 -0
- data/lib/purehtml.rb +42 -0
- data/purehtml.gemspec +29 -0
- metadata +126 -0
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../transformer"
|
|
4
|
+
require_relative "../errors"
|
|
5
|
+
|
|
6
|
+
module PureHTML
|
|
7
|
+
module Transformers
|
|
8
|
+
class SplitTransformer < Transformer
|
|
9
|
+
def self.transformer_name
|
|
10
|
+
"split"
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
def transform(params)
|
|
14
|
+
val = params.val
|
|
15
|
+
return nil if val.nil?
|
|
16
|
+
|
|
17
|
+
unless val.is_a?(String)
|
|
18
|
+
raise InvalidParseInputError.new("Split: invalid value type: #{val.class}", property: params.property, value: val)
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
delimiter = @args[0]
|
|
22
|
+
if delimiter.nil?
|
|
23
|
+
trimmed = val.strip
|
|
24
|
+
trimmed.empty? ? [] : trimmed.split(/\s+/)
|
|
25
|
+
else
|
|
26
|
+
val.split(delimiter, -1)
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../transformer"
|
|
4
|
+
|
|
5
|
+
module PureHTML
|
|
6
|
+
module Transformers
|
|
7
|
+
class TextTransformer < Transformer
|
|
8
|
+
def self.transformer_name
|
|
9
|
+
"text"
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
def transform(params)
|
|
13
|
+
params.el&.text
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
end
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../transformer"
|
|
4
|
+
require_relative "../errors"
|
|
5
|
+
|
|
6
|
+
module PureHTML
|
|
7
|
+
module Transformers
|
|
8
|
+
class TrimTransformer < Transformer
|
|
9
|
+
def self.transformer_name
|
|
10
|
+
"trim"
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
def transform(params)
|
|
14
|
+
val = params.val
|
|
15
|
+
return nil if val.nil?
|
|
16
|
+
|
|
17
|
+
raise InvalidParseInputError.new("Trim: invalid value type: #{val.class}", property: params.property, value: val) unless val.is_a?(String)
|
|
18
|
+
|
|
19
|
+
val.strip
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../transformer"
|
|
4
|
+
require_relative "../errors"
|
|
5
|
+
|
|
6
|
+
module PureHTML
|
|
7
|
+
module Transformers
|
|
8
|
+
class UpperTransformer < Transformer
|
|
9
|
+
def self.transformer_name
|
|
10
|
+
"upper"
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
def transform(params)
|
|
14
|
+
val = params.val
|
|
15
|
+
return nil if val.nil?
|
|
16
|
+
|
|
17
|
+
unless val.is_a?(String)
|
|
18
|
+
raise InvalidParseInputError.new("Upper: invalid value type: #{val.class}", property: params.property, value: val)
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
val.upcase
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "uri"
|
|
4
|
+
require_relative "../transformer"
|
|
5
|
+
require_relative "../errors"
|
|
6
|
+
|
|
7
|
+
module PureHTML
|
|
8
|
+
module Transformers
|
|
9
|
+
class UrlQueryParamTransformer < Transformer
|
|
10
|
+
def self.transformer_name
|
|
11
|
+
"urlQueryParam"
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def transform(params)
|
|
15
|
+
val = params.val
|
|
16
|
+
unless val.is_a?(String)
|
|
17
|
+
raise InvalidParseInputError.new("urlQueryParam: expected string. Got #{val.class}", property: params.property, value: val)
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
uri = URI.parse(val)
|
|
21
|
+
unless uri.query
|
|
22
|
+
return @args.empty? ? {} : (@args.length == 1 ? nil : @args.map { |a| [a, nil] }.to_h)
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
pairs = URI.decode_www_form(uri.query)
|
|
26
|
+
if @args.empty?
|
|
27
|
+
pairs.to_h
|
|
28
|
+
elsif @args.length == 1
|
|
29
|
+
pairs.find { |k, _| k == @args[0] }&.last
|
|
30
|
+
else
|
|
31
|
+
@args.map { |arg| [arg, pairs.find { |k, _| k == arg }&.last] }.to_h
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "transformers/trim"
|
|
4
|
+
require_relative "transformers/number"
|
|
5
|
+
require_relative "transformers/attr"
|
|
6
|
+
require_relative "transformers/html"
|
|
7
|
+
require_relative "transformers/text"
|
|
8
|
+
require_relative "transformers/exists"
|
|
9
|
+
require_relative "transformers/length"
|
|
10
|
+
require_relative "transformers/json"
|
|
11
|
+
require_relative "transformers/jsonpath"
|
|
12
|
+
require_relative "transformers/resolve"
|
|
13
|
+
require_relative "transformers/url_query_param"
|
|
14
|
+
require_relative "transformers/remove_url_query_param"
|
|
15
|
+
require_relative "transformers/remove_last_path_section"
|
|
16
|
+
require_relative "transformers/replace"
|
|
17
|
+
require_relative "transformers/split"
|
|
18
|
+
require_relative "transformers/join"
|
|
19
|
+
require_relative "transformers/lower"
|
|
20
|
+
require_relative "transformers/upper"
|
|
21
|
+
require_relative "transformers/capitalize"
|
|
22
|
+
require_relative "transformers/boolean"
|
|
23
|
+
|
|
24
|
+
module PureHTML
|
|
25
|
+
module Transformers
|
|
26
|
+
REGISTRY = {
|
|
27
|
+
"trim" => TrimTransformer,
|
|
28
|
+
"number" => NumberTransformer,
|
|
29
|
+
"attr" => AttrTransformer,
|
|
30
|
+
"html" => HtmlTransformer,
|
|
31
|
+
"text" => TextTransformer,
|
|
32
|
+
"exists" => ExistsTransformer,
|
|
33
|
+
"length" => LengthTransformer,
|
|
34
|
+
"json" => JsonTransformer,
|
|
35
|
+
"jsonpath" => JsonpathTransformer,
|
|
36
|
+
"resolve" => ResolveTransformer,
|
|
37
|
+
"urlQueryParam" => UrlQueryParamTransformer,
|
|
38
|
+
"removeUrlQueryParam" => RemoveUrlQueryParamTransformer,
|
|
39
|
+
"removeLastPathSection" => RemoveLastPathSectionTransformer,
|
|
40
|
+
"replace" => ReplaceTransformer,
|
|
41
|
+
"split" => SplitTransformer,
|
|
42
|
+
"join" => JoinTransformer,
|
|
43
|
+
"lower" => LowerTransformer,
|
|
44
|
+
"upper" => UpperTransformer,
|
|
45
|
+
"capitalize" => CapitalizeTransformer,
|
|
46
|
+
"boolean" => BooleanTransformer
|
|
47
|
+
}.freeze
|
|
48
|
+
|
|
49
|
+
def self.get(name)
|
|
50
|
+
REGISTRY[name]
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|
data/lib/purehtml.rb
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "purehtml/version"
|
|
4
|
+
require_relative "purehtml/errors"
|
|
5
|
+
require_relative "purehtml/extract_params"
|
|
6
|
+
require_relative "purehtml/backend"
|
|
7
|
+
require_relative "purehtml/transformer"
|
|
8
|
+
require_relative "purehtml/transformer_factory"
|
|
9
|
+
require_relative "purehtml/transformers"
|
|
10
|
+
require_relative "purehtml/config"
|
|
11
|
+
require_relative "purehtml/configs/with_selector"
|
|
12
|
+
require_relative "purehtml/configs/primitive_value_config"
|
|
13
|
+
require_relative "purehtml/configs/object_config"
|
|
14
|
+
require_relative "purehtml/configs/array_config"
|
|
15
|
+
require_relative "purehtml/configs/union_config"
|
|
16
|
+
require_relative "purehtml/configs/constant_config"
|
|
17
|
+
require_relative "purehtml/configs/default_value_config"
|
|
18
|
+
require_relative "purehtml/config_factory"
|
|
19
|
+
|
|
20
|
+
module PureHTML
|
|
21
|
+
class << self
|
|
22
|
+
attr_writer :backend
|
|
23
|
+
|
|
24
|
+
def backend
|
|
25
|
+
@backend ||= NokogiriBackend.new
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def extract(config, html, url = "", backend: nil)
|
|
29
|
+
active_backend = backend || self.backend
|
|
30
|
+
doc = html.is_a?(Document) ? html : active_backend.load(html)
|
|
31
|
+
params = ExtractParams.new(
|
|
32
|
+
doc: doc,
|
|
33
|
+
el: doc.root,
|
|
34
|
+
url: url,
|
|
35
|
+
property: "",
|
|
36
|
+
val: nil,
|
|
37
|
+
element_already_matched: false
|
|
38
|
+
)
|
|
39
|
+
config.extract(params)
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
end
|
data/purehtml.gemspec
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "lib/purehtml/version"
|
|
4
|
+
|
|
5
|
+
Gem::Specification.new do |spec|
|
|
6
|
+
spec.name = "purehtml"
|
|
7
|
+
spec.version = PureHTML::VERSION
|
|
8
|
+
spec.authors = ["Fatih Kaya"]
|
|
9
|
+
spec.email = ["1994274@gmail.com"]
|
|
10
|
+
|
|
11
|
+
spec.summary = "Declarative HTML parsing and data extraction library for Ruby"
|
|
12
|
+
spec.description = "PureHTML is a parsing specification and library for extracting JSON from HTML documents using simple YAML schemas."
|
|
13
|
+
spec.homepage = "https://purescraps.github.io/purehtml/"
|
|
14
|
+
spec.license = "MIT"
|
|
15
|
+
spec.required_ruby_version = ">= 2.6.0"
|
|
16
|
+
|
|
17
|
+
spec.metadata["homepage_uri"] = spec.homepage
|
|
18
|
+
spec.metadata["source_code_uri"] = "https://github.com/purescraps/purehtml"
|
|
19
|
+
|
|
20
|
+
spec.files = Dir.chdir(File.expand_path(__dir__)) do
|
|
21
|
+
Dir.glob("{lib}/**/*") + %w[README.md LICENSE purehtml.gemspec Rakefile]
|
|
22
|
+
end
|
|
23
|
+
spec.require_paths = ["lib"]
|
|
24
|
+
|
|
25
|
+
spec.add_dependency "nokogiri", ">= 1.10.0"
|
|
26
|
+
|
|
27
|
+
spec.add_development_dependency "rake", ">= 12.0"
|
|
28
|
+
spec.add_development_dependency "minitest", ">= 5.0"
|
|
29
|
+
end
|
metadata
ADDED
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
|
2
|
+
name: purehtml
|
|
3
|
+
version: !ruby/object:Gem::Version
|
|
4
|
+
version: 0.1.0
|
|
5
|
+
platform: ruby
|
|
6
|
+
authors:
|
|
7
|
+
- Fatih Kaya
|
|
8
|
+
bindir: bin
|
|
9
|
+
cert_chain: []
|
|
10
|
+
date: 1980-01-02 00:00:00.000000000 Z
|
|
11
|
+
dependencies:
|
|
12
|
+
- !ruby/object:Gem::Dependency
|
|
13
|
+
name: nokogiri
|
|
14
|
+
requirement: !ruby/object:Gem::Requirement
|
|
15
|
+
requirements:
|
|
16
|
+
- - ">="
|
|
17
|
+
- !ruby/object:Gem::Version
|
|
18
|
+
version: 1.10.0
|
|
19
|
+
type: :runtime
|
|
20
|
+
prerelease: false
|
|
21
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
22
|
+
requirements:
|
|
23
|
+
- - ">="
|
|
24
|
+
- !ruby/object:Gem::Version
|
|
25
|
+
version: 1.10.0
|
|
26
|
+
- !ruby/object:Gem::Dependency
|
|
27
|
+
name: rake
|
|
28
|
+
requirement: !ruby/object:Gem::Requirement
|
|
29
|
+
requirements:
|
|
30
|
+
- - ">="
|
|
31
|
+
- !ruby/object:Gem::Version
|
|
32
|
+
version: '12.0'
|
|
33
|
+
type: :development
|
|
34
|
+
prerelease: false
|
|
35
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
36
|
+
requirements:
|
|
37
|
+
- - ">="
|
|
38
|
+
- !ruby/object:Gem::Version
|
|
39
|
+
version: '12.0'
|
|
40
|
+
- !ruby/object:Gem::Dependency
|
|
41
|
+
name: minitest
|
|
42
|
+
requirement: !ruby/object:Gem::Requirement
|
|
43
|
+
requirements:
|
|
44
|
+
- - ">="
|
|
45
|
+
- !ruby/object:Gem::Version
|
|
46
|
+
version: '5.0'
|
|
47
|
+
type: :development
|
|
48
|
+
prerelease: false
|
|
49
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
50
|
+
requirements:
|
|
51
|
+
- - ">="
|
|
52
|
+
- !ruby/object:Gem::Version
|
|
53
|
+
version: '5.0'
|
|
54
|
+
description: PureHTML is a parsing specification and library for extracting JSON from
|
|
55
|
+
HTML documents using simple YAML schemas.
|
|
56
|
+
email:
|
|
57
|
+
- 1994274@gmail.com
|
|
58
|
+
executables: []
|
|
59
|
+
extensions: []
|
|
60
|
+
extra_rdoc_files: []
|
|
61
|
+
files:
|
|
62
|
+
- LICENSE
|
|
63
|
+
- README.md
|
|
64
|
+
- Rakefile
|
|
65
|
+
- lib/purehtml.rb
|
|
66
|
+
- lib/purehtml/backend.rb
|
|
67
|
+
- lib/purehtml/config.rb
|
|
68
|
+
- lib/purehtml/config_factory.rb
|
|
69
|
+
- lib/purehtml/configs/array_config.rb
|
|
70
|
+
- lib/purehtml/configs/constant_config.rb
|
|
71
|
+
- lib/purehtml/configs/default_value_config.rb
|
|
72
|
+
- lib/purehtml/configs/object_config.rb
|
|
73
|
+
- lib/purehtml/configs/primitive_value_config.rb
|
|
74
|
+
- lib/purehtml/configs/union_config.rb
|
|
75
|
+
- lib/purehtml/configs/with_selector.rb
|
|
76
|
+
- lib/purehtml/errors.rb
|
|
77
|
+
- lib/purehtml/extract_params.rb
|
|
78
|
+
- lib/purehtml/transformer.rb
|
|
79
|
+
- lib/purehtml/transformer_factory.rb
|
|
80
|
+
- lib/purehtml/transformers.rb
|
|
81
|
+
- lib/purehtml/transformers/attr.rb
|
|
82
|
+
- lib/purehtml/transformers/boolean.rb
|
|
83
|
+
- lib/purehtml/transformers/capitalize.rb
|
|
84
|
+
- lib/purehtml/transformers/exists.rb
|
|
85
|
+
- lib/purehtml/transformers/html.rb
|
|
86
|
+
- lib/purehtml/transformers/join.rb
|
|
87
|
+
- lib/purehtml/transformers/json.rb
|
|
88
|
+
- lib/purehtml/transformers/jsonpath.rb
|
|
89
|
+
- lib/purehtml/transformers/length.rb
|
|
90
|
+
- lib/purehtml/transformers/lower.rb
|
|
91
|
+
- lib/purehtml/transformers/number.rb
|
|
92
|
+
- lib/purehtml/transformers/remove_last_path_section.rb
|
|
93
|
+
- lib/purehtml/transformers/remove_url_query_param.rb
|
|
94
|
+
- lib/purehtml/transformers/replace.rb
|
|
95
|
+
- lib/purehtml/transformers/resolve.rb
|
|
96
|
+
- lib/purehtml/transformers/split.rb
|
|
97
|
+
- lib/purehtml/transformers/text.rb
|
|
98
|
+
- lib/purehtml/transformers/trim.rb
|
|
99
|
+
- lib/purehtml/transformers/upper.rb
|
|
100
|
+
- lib/purehtml/transformers/url_query_param.rb
|
|
101
|
+
- lib/purehtml/version.rb
|
|
102
|
+
- purehtml.gemspec
|
|
103
|
+
homepage: https://purescraps.github.io/purehtml/
|
|
104
|
+
licenses:
|
|
105
|
+
- MIT
|
|
106
|
+
metadata:
|
|
107
|
+
homepage_uri: https://purescraps.github.io/purehtml/
|
|
108
|
+
source_code_uri: https://github.com/purescraps/purehtml
|
|
109
|
+
rdoc_options: []
|
|
110
|
+
require_paths:
|
|
111
|
+
- lib
|
|
112
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
|
113
|
+
requirements:
|
|
114
|
+
- - ">="
|
|
115
|
+
- !ruby/object:Gem::Version
|
|
116
|
+
version: 2.6.0
|
|
117
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
118
|
+
requirements:
|
|
119
|
+
- - ">="
|
|
120
|
+
- !ruby/object:Gem::Version
|
|
121
|
+
version: '0'
|
|
122
|
+
requirements: []
|
|
123
|
+
rubygems_version: 4.0.20
|
|
124
|
+
specification_version: 4
|
|
125
|
+
summary: Declarative HTML parsing and data extraction library for Ruby
|
|
126
|
+
test_files: []
|