purehtml 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +7 -0
- data/README.md +173 -0
- data/Rakefile +12 -0
- data/lib/purehtml/backend.rb +120 -0
- data/lib/purehtml/config.rb +9 -0
- data/lib/purehtml/config_factory.rb +129 -0
- data/lib/purehtml/configs/array_config.rb +32 -0
- data/lib/purehtml/configs/constant_config.rb +26 -0
- data/lib/purehtml/configs/default_value_config.rb +22 -0
- data/lib/purehtml/configs/object_config.rb +29 -0
- data/lib/purehtml/configs/primitive_value_config.rb +36 -0
- data/lib/purehtml/configs/union_config.rb +47 -0
- data/lib/purehtml/configs/with_selector.rb +33 -0
- data/lib/purehtml/errors.rb +24 -0
- data/lib/purehtml/extract_params.rb +27 -0
- data/lib/purehtml/transformer.rb +19 -0
- data/lib/purehtml/transformer_factory.rb +76 -0
- data/lib/purehtml/transformers/attr.rb +28 -0
- data/lib/purehtml/transformers/boolean.rb +25 -0
- data/lib/purehtml/transformers/capitalize.rb +27 -0
- data/lib/purehtml/transformers/exists.rb +17 -0
- data/lib/purehtml/transformers/html.rb +17 -0
- data/lib/purehtml/transformers/join.rb +26 -0
- data/lib/purehtml/transformers/json.rb +22 -0
- data/lib/purehtml/transformers/jsonpath.rb +139 -0
- data/lib/purehtml/transformers/length.rb +23 -0
- data/lib/purehtml/transformers/lower.rb +25 -0
- data/lib/purehtml/transformers/number.rb +37 -0
- data/lib/purehtml/transformers/remove_last_path_section.rb +40 -0
- data/lib/purehtml/transformers/remove_url_query_param.rb +36 -0
- data/lib/purehtml/transformers/replace.rb +36 -0
- data/lib/purehtml/transformers/resolve.rb +24 -0
- data/lib/purehtml/transformers/split.rb +31 -0
- data/lib/purehtml/transformers/text.rb +17 -0
- data/lib/purehtml/transformers/trim.rb +23 -0
- data/lib/purehtml/transformers/upper.rb +25 -0
- data/lib/purehtml/transformers/url_query_param.rb +36 -0
- data/lib/purehtml/transformers.rb +53 -0
- data/lib/purehtml/version.rb +5 -0
- data/lib/purehtml.rb +42 -0
- data/purehtml.gemspec +29 -0
- metadata +126 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: a768efe71c391b97100948faf0ac5de6131971e9db477b8c1711aeb4c847a506
|
|
4
|
+
data.tar.gz: f18783bfa9e3d67dbd7280dabde3d395a0224e52b71da21cdde76ac94787c9b5
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: 5191e380c661c485c89cd2d066bfb783d637618fa3b72216af3bd9fb8ea4c466d84abb41928db8d1df76ed93530800274d3a0feff9dded233f1e6690370a9a8d
|
|
7
|
+
data.tar.gz: 366d5960a80c655c64641a644d1fffa65c2d35809d8fc474676612d65acebb93c7570683a15010573ab2daf47b5379b7fb5118cb1083fba59020789a5e069cd0
|
data/LICENSE
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
Copyright 2024 Fatih Kaya <1994274@gmail.com>
|
|
2
|
+
|
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
|
4
|
+
|
|
5
|
+
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
|
6
|
+
|
|
7
|
+
THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
# PureHTML for Ruby
|
|
2
|
+
|
|
3
|
+
PureHTML is a declarative HTML parsing and extraction library for Ruby. Extract structured JSON/data from HTML documents using simple YAML schemas.
|
|
4
|
+
|
|
5
|
+
- [Documentation & Playground](https://purescraps.github.io/purehtml/)
|
|
6
|
+
|
|
7
|
+
## Features
|
|
8
|
+
|
|
9
|
+
- **Declarative Schemas**: Simple YAML definitions for HTML data extraction.
|
|
10
|
+
- **Pluggable Backends**: Built on top of [Nokogiri](https://nokogiri.org).
|
|
11
|
+
- **Extensible Transformers**: Value transformation chain (e.g. `trim`, `number`, `attr`, `html`, `json`, `jsonpath`, `split`, `join`, `replace`, etc.).
|
|
12
|
+
- **Complex Structures**: Supports primitives, nested objects, arrays, unions, constants, and default values.
|
|
13
|
+
|
|
14
|
+
## Installation
|
|
15
|
+
|
|
16
|
+
Add this line to your application's Gemfile:
|
|
17
|
+
|
|
18
|
+
```ruby
|
|
19
|
+
gem 'purehtml'
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
Or install it yourself as:
|
|
23
|
+
|
|
24
|
+
```bash
|
|
25
|
+
gem install purehtml
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
## Quick Start
|
|
29
|
+
|
|
30
|
+
```ruby
|
|
31
|
+
require 'purehtml'
|
|
32
|
+
|
|
33
|
+
yaml_string = <<~YAML
|
|
34
|
+
selector: span
|
|
35
|
+
type: array
|
|
36
|
+
items: {}
|
|
37
|
+
YAML
|
|
38
|
+
|
|
39
|
+
html_string = <<~HTML
|
|
40
|
+
<div>
|
|
41
|
+
<span>a</span>
|
|
42
|
+
<span>b</span>
|
|
43
|
+
<span>c</span>
|
|
44
|
+
</div>
|
|
45
|
+
HTML
|
|
46
|
+
|
|
47
|
+
config = PureHTML::ConfigFactory.from_yaml(yaml_string)
|
|
48
|
+
result = PureHTML.extract(config, html_string, "http://example.com")
|
|
49
|
+
|
|
50
|
+
puts result.inspect
|
|
51
|
+
# => ["a", "b", "c"]
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
## Configuration Types
|
|
55
|
+
|
|
56
|
+
### Primitive Value
|
|
57
|
+
|
|
58
|
+
Extract a single value:
|
|
59
|
+
|
|
60
|
+
```yaml
|
|
61
|
+
selector: .title
|
|
62
|
+
transform: trim
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
### Object
|
|
66
|
+
|
|
67
|
+
Extract multiple properties:
|
|
68
|
+
|
|
69
|
+
```yaml
|
|
70
|
+
selector: body
|
|
71
|
+
properties:
|
|
72
|
+
title:
|
|
73
|
+
selector: .title
|
|
74
|
+
transform: trim
|
|
75
|
+
description:
|
|
76
|
+
selector: .desc
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
### Array
|
|
80
|
+
|
|
81
|
+
Extract multiple items:
|
|
82
|
+
|
|
83
|
+
```yaml
|
|
84
|
+
selector: ul.items li
|
|
85
|
+
type: array
|
|
86
|
+
items:
|
|
87
|
+
properties:
|
|
88
|
+
name:
|
|
89
|
+
selector: .name
|
|
90
|
+
transform: trim
|
|
91
|
+
price:
|
|
92
|
+
selector: .price
|
|
93
|
+
type: number
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
### Union
|
|
97
|
+
|
|
98
|
+
Try multiple extraction strategies until one matches:
|
|
99
|
+
|
|
100
|
+
```yaml
|
|
101
|
+
union:
|
|
102
|
+
- selector: .modern-date
|
|
103
|
+
- selector: .legacy-date
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
### Constant
|
|
107
|
+
|
|
108
|
+
Return a constant value:
|
|
109
|
+
|
|
110
|
+
```yaml
|
|
111
|
+
selector: .featured-badge
|
|
112
|
+
constant: true
|
|
113
|
+
default: false
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
### Default Fallback
|
|
117
|
+
|
|
118
|
+
Provide a fallback value when a selector does not match:
|
|
119
|
+
|
|
120
|
+
```yaml
|
|
121
|
+
selector: .non-existent
|
|
122
|
+
default: "N/A"
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
### Fallback Selectors
|
|
126
|
+
|
|
127
|
+
Specify an array of fallback selectors for resilient scraping across redesigns or A/B tests:
|
|
128
|
+
|
|
129
|
+
```yaml
|
|
130
|
+
selector:
|
|
131
|
+
- .flash-deal-price
|
|
132
|
+
- .sale-price
|
|
133
|
+
- .regular-price
|
|
134
|
+
transform: trim
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
The schema also supports selector objects with `sampleHTMLs`:
|
|
138
|
+
|
|
139
|
+
```yaml
|
|
140
|
+
selector:
|
|
141
|
+
- selector: .promo-price
|
|
142
|
+
sampleHTMLs:
|
|
143
|
+
- tests/fixtures/promo.html
|
|
144
|
+
- selector: .standard-price
|
|
145
|
+
sampleHTMLs:
|
|
146
|
+
- tests/fixtures/standard.html
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
### Transformer Argument Quoting Rules
|
|
150
|
+
|
|
151
|
+
When transformer arguments contain spaces, commas, or regex escape sequences (e.g. `replace`, `split`, `join`, `urlQueryParam`), enclose them in single or double quotes:
|
|
152
|
+
|
|
153
|
+
```yaml
|
|
154
|
+
transform:
|
|
155
|
+
- 'replace("\\s+", " ")'
|
|
156
|
+
- 'split(", ")'
|
|
157
|
+
- 'join(" - ")'
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
Inside quotes, a backslash escapes matching quotes (`\"` or `\'`) or literal backslashes (`\\`), while regex escapes (`\s`, `\d`) pass through untouched.
|
|
161
|
+
|
|
162
|
+
## Development
|
|
163
|
+
|
|
164
|
+
Run tests with `rake`:
|
|
165
|
+
|
|
166
|
+
```bash
|
|
167
|
+
cd ruby
|
|
168
|
+
rake test
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
## License
|
|
172
|
+
|
|
173
|
+
[MIT](LICENSE)
|
data/Rakefile
ADDED
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "nokogiri"
|
|
4
|
+
|
|
5
|
+
module PureHTML
|
|
6
|
+
class Backend
|
|
7
|
+
def load(html)
|
|
8
|
+
raise NotImplementedError
|
|
9
|
+
end
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
class Document
|
|
13
|
+
def root
|
|
14
|
+
raise NotImplementedError
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def find(selector)
|
|
18
|
+
raise NotImplementedError
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
class Node
|
|
23
|
+
def attr(name = nil)
|
|
24
|
+
raise NotImplementedError
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def find(selector)
|
|
28
|
+
raise NotImplementedError
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def html
|
|
32
|
+
raise NotImplementedError
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def is_selector?(selector)
|
|
36
|
+
raise NotImplementedError
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def text
|
|
40
|
+
raise NotImplementedError
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
class NokogiriBackend < Backend
|
|
45
|
+
def load(html)
|
|
46
|
+
NokogiriDocument.new(Nokogiri::HTML(html))
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
class NokogiriDocument < Document
|
|
51
|
+
attr_reader :nokogiri_doc
|
|
52
|
+
|
|
53
|
+
def initialize(nokogiri_doc)
|
|
54
|
+
@nokogiri_doc = nokogiri_doc
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def root
|
|
58
|
+
NokogiriNode.new(@nokogiri_doc, @nokogiri_doc)
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def find(selector)
|
|
62
|
+
@nokogiri_doc.css(selector).map { |node| NokogiriNode.new(@nokogiri_doc, node) }
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
class NokogiriNode < Node
|
|
67
|
+
attr_reader :doc, :node
|
|
68
|
+
|
|
69
|
+
def initialize(doc, node)
|
|
70
|
+
@doc = doc
|
|
71
|
+
@node = node
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
def is_selector?(selector)
|
|
75
|
+
return false if @node.nil? || @node.document?
|
|
76
|
+
|
|
77
|
+
if @node.respond_to?(:matches?)
|
|
78
|
+
begin
|
|
79
|
+
return @node.matches?(selector)
|
|
80
|
+
rescue StandardError
|
|
81
|
+
# Fall through to doc.css check if matches? encounters an unsupported selector syntax
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
@doc.css(selector).include?(@node)
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
def find(selector)
|
|
89
|
+
nodes = if @node.document?
|
|
90
|
+
@doc.css(selector)
|
|
91
|
+
else
|
|
92
|
+
@node.css(selector)
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
nodes.map { |n| NokogiriNode.new(@doc, n) }
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
def text
|
|
99
|
+
return "" if @node.nil?
|
|
100
|
+
|
|
101
|
+
@node.text
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
def html
|
|
105
|
+
return nil if @node.nil?
|
|
106
|
+
|
|
107
|
+
@node.inner_html
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
def attr(name = nil)
|
|
111
|
+
return nil if @node.nil? || @node.document?
|
|
112
|
+
|
|
113
|
+
if name
|
|
114
|
+
@node[name]
|
|
115
|
+
else
|
|
116
|
+
@node.respond_to?(:attributes) ? @node.attributes.transform_values(&:value) : {}
|
|
117
|
+
end
|
|
118
|
+
end
|
|
119
|
+
end
|
|
120
|
+
end
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "yaml"
|
|
4
|
+
require_relative "configs/with_selector"
|
|
5
|
+
require_relative "configs/primitive_value_config"
|
|
6
|
+
require_relative "configs/object_config"
|
|
7
|
+
require_relative "configs/array_config"
|
|
8
|
+
require_relative "configs/union_config"
|
|
9
|
+
require_relative "configs/constant_config"
|
|
10
|
+
require_relative "configs/default_value_config"
|
|
11
|
+
require_relative "transformer_factory"
|
|
12
|
+
require_relative "errors"
|
|
13
|
+
|
|
14
|
+
module PureHTML
|
|
15
|
+
class ConfigFactory
|
|
16
|
+
def self.from_file(path)
|
|
17
|
+
from_yaml(File.read(path))
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
def self.from_yaml(yaml_or_hash)
|
|
21
|
+
hash = if yaml_or_hash.is_a?(String)
|
|
22
|
+
begin
|
|
23
|
+
YAML.safe_load(yaml_or_hash, aliases: true)
|
|
24
|
+
rescue ArgumentError
|
|
25
|
+
YAML.safe_load(yaml_or_hash)
|
|
26
|
+
end
|
|
27
|
+
else
|
|
28
|
+
yaml_or_hash
|
|
29
|
+
end
|
|
30
|
+
from_hash(hash)
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def self.from_hash(plain)
|
|
34
|
+
plain = stringify_keys(plain || {})
|
|
35
|
+
constant = plain["constant"]
|
|
36
|
+
selector_orig = plain["selector"]
|
|
37
|
+
transform_orig = plain["transform"]
|
|
38
|
+
properties = plain["properties"]
|
|
39
|
+
items = plain["items"]
|
|
40
|
+
union = plain["union"]
|
|
41
|
+
default_value = plain["default"]
|
|
42
|
+
type = plain["type"]
|
|
43
|
+
|
|
44
|
+
selector = generate_selector(selector_orig)
|
|
45
|
+
expected_type = detect_expected_type(plain)
|
|
46
|
+
transformers = generate_transform(transform_orig)
|
|
47
|
+
|
|
48
|
+
config = case expected_type
|
|
49
|
+
when "constant"
|
|
50
|
+
Configs::ConstantConfig.new(constant, selector: selector)
|
|
51
|
+
when "object"
|
|
52
|
+
prop_configs = if properties.is_a?(Hash)
|
|
53
|
+
properties.transform_keys(&:to_s).transform_values { |v| from_hash(v) }
|
|
54
|
+
else
|
|
55
|
+
{}
|
|
56
|
+
end
|
|
57
|
+
Configs::ObjectConfig.new(selector: selector, properties: prop_configs)
|
|
58
|
+
when "array"
|
|
59
|
+
Configs::ArrayConfig.new(
|
|
60
|
+
selector: selector,
|
|
61
|
+
items: items ? from_hash(items) : nil,
|
|
62
|
+
transform: transformers
|
|
63
|
+
)
|
|
64
|
+
when "union"
|
|
65
|
+
Configs::UnionConfig.new((union || []).map { |u| from_hash(u) })
|
|
66
|
+
else
|
|
67
|
+
coerced_transformers = append_type_coercion(type, transformers)
|
|
68
|
+
Configs::PrimitiveValueConfig.new(selector: selector, transform: coerced_transformers)
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
if plain.key?("default")
|
|
72
|
+
Configs::DefaultValueConfig.new(config, default_value)
|
|
73
|
+
else
|
|
74
|
+
config
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def self.generate_selector(selector)
|
|
79
|
+
if selector.is_a?(String)
|
|
80
|
+
selector.empty? ? nil : selector
|
|
81
|
+
elsif selector.is_a?(Array)
|
|
82
|
+
joined = selector.map { |s| generate_selector(s) }.compact.join(", ")
|
|
83
|
+
joined.empty? ? nil : joined
|
|
84
|
+
elsif selector.is_a?(Hash)
|
|
85
|
+
sel = selector["selector"] || selector[:selector]
|
|
86
|
+
generate_selector(sel)
|
|
87
|
+
elsif selector.nil?
|
|
88
|
+
nil
|
|
89
|
+
else
|
|
90
|
+
raise Error, "Unexpected selector type: #{selector.class}"
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def self.generate_transform(transform)
|
|
95
|
+
return nil if transform.nil?
|
|
96
|
+
|
|
97
|
+
if transform.is_a?(String)
|
|
98
|
+
[TransformerFactory.create(transform)]
|
|
99
|
+
elsif transform.is_a?(Array)
|
|
100
|
+
transform.map { |t| TransformerFactory.create(t) }
|
|
101
|
+
else
|
|
102
|
+
raise Error, "Unexpected transform type: #{transform.class}"
|
|
103
|
+
end
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
def self.append_type_coercion(type, transformers)
|
|
107
|
+
return transformers unless %w[number boolean].include?(type)
|
|
108
|
+
|
|
109
|
+
coercion = TransformerFactory.create(type)
|
|
110
|
+
(transformers || []) + [coercion]
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
def self.detect_expected_type(conf)
|
|
114
|
+
return "object" if conf.key?("properties") || conf["type"] == "object"
|
|
115
|
+
return "array" if conf.key?("items") || conf["type"] == "array"
|
|
116
|
+
return "union" if conf.key?("union")
|
|
117
|
+
return "constant" if conf.key?("constant")
|
|
118
|
+
|
|
119
|
+
"primitive"
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
def self.stringify_keys(hash)
|
|
123
|
+
return hash unless hash.is_a?(Hash)
|
|
124
|
+
|
|
125
|
+
hash.transform_keys(&:to_s)
|
|
126
|
+
end
|
|
127
|
+
private_class_method :stringify_keys
|
|
128
|
+
end
|
|
129
|
+
end
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "with_selector"
|
|
4
|
+
require_relative "primitive_value_config"
|
|
5
|
+
|
|
6
|
+
module PureHTML
|
|
7
|
+
module Configs
|
|
8
|
+
class ArrayConfig < WithSelector
|
|
9
|
+
attr_reader :items, :transform
|
|
10
|
+
|
|
11
|
+
def initialize(selector: nil, items: nil, transform: nil)
|
|
12
|
+
super(selector: selector)
|
|
13
|
+
@items = items
|
|
14
|
+
@transform = transform.nil? ? [] : (transform.is_a?(Array) ? transform : [transform])
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def extract(params)
|
|
18
|
+
matches = get_all_matches(params.el, already_matched: params.element_already_matched, include_root: false)
|
|
19
|
+
return nil if matches.nil?
|
|
20
|
+
|
|
21
|
+
matches = matches.is_a?(Array) ? matches : [matches]
|
|
22
|
+
conf = @items || PrimitiveValueConfig.new(selector: nil, transform: @transform)
|
|
23
|
+
|
|
24
|
+
matches.map.with_index do |el, i|
|
|
25
|
+
prop = params.property.empty? ? "[#{i}]" : "#{params.property}[#{i}]"
|
|
26
|
+
sub_params = params.dup_with(el: el, property: prop, element_already_matched: false)
|
|
27
|
+
conf.extract(sub_params)
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "with_selector"
|
|
4
|
+
|
|
5
|
+
module PureHTML
|
|
6
|
+
module Configs
|
|
7
|
+
class ConstantConfig < WithSelector
|
|
8
|
+
attr_reader :constant
|
|
9
|
+
|
|
10
|
+
def initialize(constant, selector: nil)
|
|
11
|
+
super(selector: selector)
|
|
12
|
+
@constant = constant
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def extract(params)
|
|
16
|
+
if @selector
|
|
17
|
+
matches = get_all_matches(params.el, already_matched: false, include_root: true)
|
|
18
|
+
if matches.is_a?(Array) && matches.empty?
|
|
19
|
+
return nil
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
@constant
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
end
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../config"
|
|
4
|
+
|
|
5
|
+
module PureHTML
|
|
6
|
+
module Configs
|
|
7
|
+
class DefaultValueConfig < Config
|
|
8
|
+
attr_reader :inner, :default_value
|
|
9
|
+
|
|
10
|
+
def initialize(inner, default_value)
|
|
11
|
+
super()
|
|
12
|
+
@inner = inner
|
|
13
|
+
@default_value = default_value
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def extract(params)
|
|
17
|
+
val = @inner.extract(params)
|
|
18
|
+
val.nil? ? @default_value : val
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
end
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "with_selector"
|
|
4
|
+
|
|
5
|
+
module PureHTML
|
|
6
|
+
module Configs
|
|
7
|
+
class ObjectConfig < WithSelector
|
|
8
|
+
attr_reader :properties
|
|
9
|
+
|
|
10
|
+
def initialize(selector: nil, properties: {})
|
|
11
|
+
super(selector: selector)
|
|
12
|
+
@properties = properties || {}
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def extract(params)
|
|
16
|
+
el = get_first_match(params.el, already_matched: params.element_already_matched, include_root: false)
|
|
17
|
+
return nil if el.nil?
|
|
18
|
+
|
|
19
|
+
result = {}
|
|
20
|
+
@properties.each do |key, cfg|
|
|
21
|
+
prop = params.property.empty? ? key.to_s : "#{params.property}.#{key}"
|
|
22
|
+
sub_params = params.dup_with(el: el, property: prop, element_already_matched: false)
|
|
23
|
+
result[key.to_s] = cfg.extract(sub_params)
|
|
24
|
+
end
|
|
25
|
+
result
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
end
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "with_selector"
|
|
4
|
+
|
|
5
|
+
module PureHTML
|
|
6
|
+
module Configs
|
|
7
|
+
class PrimitiveValueConfig < WithSelector
|
|
8
|
+
attr_reader :transform
|
|
9
|
+
|
|
10
|
+
def initialize(selector: nil, transform: nil)
|
|
11
|
+
super(selector: selector)
|
|
12
|
+
@transform = transform.nil? ? [] : (transform.is_a?(Array) ? transform : [transform])
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def extract(params)
|
|
16
|
+
el = get_first_match(params.el, already_matched: params.element_already_matched, include_root: false)
|
|
17
|
+
val = el ? el.text : nil
|
|
18
|
+
|
|
19
|
+
return val if @transform.empty?
|
|
20
|
+
|
|
21
|
+
transform_val(@transform, params.dup_with(el: el, val: val))
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
private
|
|
25
|
+
|
|
26
|
+
def transform_val(transformers, params)
|
|
27
|
+
current_val = params.val
|
|
28
|
+
transformers.each do |tr|
|
|
29
|
+
p = params.dup_with(val: current_val)
|
|
30
|
+
current_val = tr.transform(p)
|
|
31
|
+
end
|
|
32
|
+
current_val
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../config"
|
|
4
|
+
require_relative "with_selector"
|
|
5
|
+
|
|
6
|
+
module PureHTML
|
|
7
|
+
module Configs
|
|
8
|
+
class UnionConfig < Config
|
|
9
|
+
attr_reader :configs
|
|
10
|
+
|
|
11
|
+
def initialize(configs)
|
|
12
|
+
super()
|
|
13
|
+
@configs = configs || []
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def extract(params)
|
|
17
|
+
parent = params.el
|
|
18
|
+
@configs.each do |config|
|
|
19
|
+
if config.is_a?(WithSelector)
|
|
20
|
+
matches = config.get_all_matches(parent, already_matched: false, include_root: true)
|
|
21
|
+
next if matches.nil? || (matches.is_a?(Array) && matches.empty?)
|
|
22
|
+
|
|
23
|
+
is_array = matches.is_a?(Array)
|
|
24
|
+
matched_el = if is_array
|
|
25
|
+
matches.length > 1 ? parent : matches[0]
|
|
26
|
+
else
|
|
27
|
+
matches
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
element_already_matched = !is_array || matches.length == 1
|
|
31
|
+
|
|
32
|
+
sub_params = params.dup_with(
|
|
33
|
+
el: matched_el,
|
|
34
|
+
element_already_matched: element_already_matched
|
|
35
|
+
)
|
|
36
|
+
return config.extract(sub_params)
|
|
37
|
+
else
|
|
38
|
+
sub_params = params.dup_with(el: parent)
|
|
39
|
+
val = config.extract(sub_params)
|
|
40
|
+
return val unless val.nil?
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
nil
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
end
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../config"
|
|
4
|
+
|
|
5
|
+
module PureHTML
|
|
6
|
+
module Configs
|
|
7
|
+
class WithSelector < Config
|
|
8
|
+
attr_reader :selector
|
|
9
|
+
|
|
10
|
+
def initialize(selector: nil)
|
|
11
|
+
super()
|
|
12
|
+
@selector = selector
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def get_all_matches(node, already_matched: false, include_root: false)
|
|
16
|
+
return node if already_matched
|
|
17
|
+
return node if @selector.nil?
|
|
18
|
+
|
|
19
|
+
if include_root && node.is_selector?(@selector)
|
|
20
|
+
return node
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
matches = node.find(@selector)
|
|
24
|
+
matches.nil? ? [] : matches
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def get_first_match(node, already_matched: false, include_root: false)
|
|
28
|
+
matches = get_all_matches(node, already_matched: already_matched, include_root: include_root)
|
|
29
|
+
matches.is_a?(Array) ? matches.first : matches
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|