purehtml 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +7 -0
  3. data/README.md +173 -0
  4. data/Rakefile +12 -0
  5. data/lib/purehtml/backend.rb +120 -0
  6. data/lib/purehtml/config.rb +9 -0
  7. data/lib/purehtml/config_factory.rb +129 -0
  8. data/lib/purehtml/configs/array_config.rb +32 -0
  9. data/lib/purehtml/configs/constant_config.rb +26 -0
  10. data/lib/purehtml/configs/default_value_config.rb +22 -0
  11. data/lib/purehtml/configs/object_config.rb +29 -0
  12. data/lib/purehtml/configs/primitive_value_config.rb +36 -0
  13. data/lib/purehtml/configs/union_config.rb +47 -0
  14. data/lib/purehtml/configs/with_selector.rb +33 -0
  15. data/lib/purehtml/errors.rb +24 -0
  16. data/lib/purehtml/extract_params.rb +27 -0
  17. data/lib/purehtml/transformer.rb +19 -0
  18. data/lib/purehtml/transformer_factory.rb +76 -0
  19. data/lib/purehtml/transformers/attr.rb +28 -0
  20. data/lib/purehtml/transformers/boolean.rb +25 -0
  21. data/lib/purehtml/transformers/capitalize.rb +27 -0
  22. data/lib/purehtml/transformers/exists.rb +17 -0
  23. data/lib/purehtml/transformers/html.rb +17 -0
  24. data/lib/purehtml/transformers/join.rb +26 -0
  25. data/lib/purehtml/transformers/json.rb +22 -0
  26. data/lib/purehtml/transformers/jsonpath.rb +139 -0
  27. data/lib/purehtml/transformers/length.rb +23 -0
  28. data/lib/purehtml/transformers/lower.rb +25 -0
  29. data/lib/purehtml/transformers/number.rb +37 -0
  30. data/lib/purehtml/transformers/remove_last_path_section.rb +40 -0
  31. data/lib/purehtml/transformers/remove_url_query_param.rb +36 -0
  32. data/lib/purehtml/transformers/replace.rb +36 -0
  33. data/lib/purehtml/transformers/resolve.rb +24 -0
  34. data/lib/purehtml/transformers/split.rb +31 -0
  35. data/lib/purehtml/transformers/text.rb +17 -0
  36. data/lib/purehtml/transformers/trim.rb +23 -0
  37. data/lib/purehtml/transformers/upper.rb +25 -0
  38. data/lib/purehtml/transformers/url_query_param.rb +36 -0
  39. data/lib/purehtml/transformers.rb +53 -0
  40. data/lib/purehtml/version.rb +5 -0
  41. data/lib/purehtml.rb +42 -0
  42. data/purehtml.gemspec +29 -0
  43. metadata +126 -0
@@ -0,0 +1,31 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../transformer"
4
+ require_relative "../errors"
5
+
6
+ module PureHTML
7
+ module Transformers
8
+ class SplitTransformer < Transformer
9
+ def self.transformer_name
10
+ "split"
11
+ end
12
+
13
+ def transform(params)
14
+ val = params.val
15
+ return nil if val.nil?
16
+
17
+ unless val.is_a?(String)
18
+ raise InvalidParseInputError.new("Split: invalid value type: #{val.class}", property: params.property, value: val)
19
+ end
20
+
21
+ delimiter = @args[0]
22
+ if delimiter.nil?
23
+ trimmed = val.strip
24
+ trimmed.empty? ? [] : trimmed.split(/\s+/)
25
+ else
26
+ val.split(delimiter, -1)
27
+ end
28
+ end
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,17 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../transformer"
4
+
5
+ module PureHTML
6
+ module Transformers
7
+ class TextTransformer < Transformer
8
+ def self.transformer_name
9
+ "text"
10
+ end
11
+
12
+ def transform(params)
13
+ params.el&.text
14
+ end
15
+ end
16
+ end
17
+ end
@@ -0,0 +1,23 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../transformer"
4
+ require_relative "../errors"
5
+
6
+ module PureHTML
7
+ module Transformers
8
+ class TrimTransformer < Transformer
9
+ def self.transformer_name
10
+ "trim"
11
+ end
12
+
13
+ def transform(params)
14
+ val = params.val
15
+ return nil if val.nil?
16
+
17
+ raise InvalidParseInputError.new("Trim: invalid value type: #{val.class}", property: params.property, value: val) unless val.is_a?(String)
18
+
19
+ val.strip
20
+ end
21
+ end
22
+ end
23
+ end
@@ -0,0 +1,25 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../transformer"
4
+ require_relative "../errors"
5
+
6
+ module PureHTML
7
+ module Transformers
8
+ class UpperTransformer < Transformer
9
+ def self.transformer_name
10
+ "upper"
11
+ end
12
+
13
+ def transform(params)
14
+ val = params.val
15
+ return nil if val.nil?
16
+
17
+ unless val.is_a?(String)
18
+ raise InvalidParseInputError.new("Upper: invalid value type: #{val.class}", property: params.property, value: val)
19
+ end
20
+
21
+ val.upcase
22
+ end
23
+ end
24
+ end
25
+ end
@@ -0,0 +1,36 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "uri"
4
+ require_relative "../transformer"
5
+ require_relative "../errors"
6
+
7
+ module PureHTML
8
+ module Transformers
9
+ class UrlQueryParamTransformer < Transformer
10
+ def self.transformer_name
11
+ "urlQueryParam"
12
+ end
13
+
14
+ def transform(params)
15
+ val = params.val
16
+ unless val.is_a?(String)
17
+ raise InvalidParseInputError.new("urlQueryParam: expected string. Got #{val.class}", property: params.property, value: val)
18
+ end
19
+
20
+ uri = URI.parse(val)
21
+ unless uri.query
22
+ return @args.empty? ? {} : (@args.length == 1 ? nil : @args.map { |a| [a, nil] }.to_h)
23
+ end
24
+
25
+ pairs = URI.decode_www_form(uri.query)
26
+ if @args.empty?
27
+ pairs.to_h
28
+ elsif @args.length == 1
29
+ pairs.find { |k, _| k == @args[0] }&.last
30
+ else
31
+ @args.map { |arg| [arg, pairs.find { |k, _| k == arg }&.last] }.to_h
32
+ end
33
+ end
34
+ end
35
+ end
36
+ end
@@ -0,0 +1,53 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "transformers/trim"
4
+ require_relative "transformers/number"
5
+ require_relative "transformers/attr"
6
+ require_relative "transformers/html"
7
+ require_relative "transformers/text"
8
+ require_relative "transformers/exists"
9
+ require_relative "transformers/length"
10
+ require_relative "transformers/json"
11
+ require_relative "transformers/jsonpath"
12
+ require_relative "transformers/resolve"
13
+ require_relative "transformers/url_query_param"
14
+ require_relative "transformers/remove_url_query_param"
15
+ require_relative "transformers/remove_last_path_section"
16
+ require_relative "transformers/replace"
17
+ require_relative "transformers/split"
18
+ require_relative "transformers/join"
19
+ require_relative "transformers/lower"
20
+ require_relative "transformers/upper"
21
+ require_relative "transformers/capitalize"
22
+ require_relative "transformers/boolean"
23
+
24
+ module PureHTML
25
+ module Transformers
26
+ REGISTRY = {
27
+ "trim" => TrimTransformer,
28
+ "number" => NumberTransformer,
29
+ "attr" => AttrTransformer,
30
+ "html" => HtmlTransformer,
31
+ "text" => TextTransformer,
32
+ "exists" => ExistsTransformer,
33
+ "length" => LengthTransformer,
34
+ "json" => JsonTransformer,
35
+ "jsonpath" => JsonpathTransformer,
36
+ "resolve" => ResolveTransformer,
37
+ "urlQueryParam" => UrlQueryParamTransformer,
38
+ "removeUrlQueryParam" => RemoveUrlQueryParamTransformer,
39
+ "removeLastPathSection" => RemoveLastPathSectionTransformer,
40
+ "replace" => ReplaceTransformer,
41
+ "split" => SplitTransformer,
42
+ "join" => JoinTransformer,
43
+ "lower" => LowerTransformer,
44
+ "upper" => UpperTransformer,
45
+ "capitalize" => CapitalizeTransformer,
46
+ "boolean" => BooleanTransformer
47
+ }.freeze
48
+
49
+ def self.get(name)
50
+ REGISTRY[name]
51
+ end
52
+ end
53
+ end
@@ -0,0 +1,5 @@
1
+ # frozen_string_literal: true
2
+
3
+ module PureHTML
4
+ VERSION = "0.1.0"
5
+ end
data/lib/purehtml.rb ADDED
@@ -0,0 +1,42 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "purehtml/version"
4
+ require_relative "purehtml/errors"
5
+ require_relative "purehtml/extract_params"
6
+ require_relative "purehtml/backend"
7
+ require_relative "purehtml/transformer"
8
+ require_relative "purehtml/transformer_factory"
9
+ require_relative "purehtml/transformers"
10
+ require_relative "purehtml/config"
11
+ require_relative "purehtml/configs/with_selector"
12
+ require_relative "purehtml/configs/primitive_value_config"
13
+ require_relative "purehtml/configs/object_config"
14
+ require_relative "purehtml/configs/array_config"
15
+ require_relative "purehtml/configs/union_config"
16
+ require_relative "purehtml/configs/constant_config"
17
+ require_relative "purehtml/configs/default_value_config"
18
+ require_relative "purehtml/config_factory"
19
+
20
+ module PureHTML
21
+ class << self
22
+ attr_writer :backend
23
+
24
+ def backend
25
+ @backend ||= NokogiriBackend.new
26
+ end
27
+
28
+ def extract(config, html, url = "", backend: nil)
29
+ active_backend = backend || self.backend
30
+ doc = html.is_a?(Document) ? html : active_backend.load(html)
31
+ params = ExtractParams.new(
32
+ doc: doc,
33
+ el: doc.root,
34
+ url: url,
35
+ property: "",
36
+ val: nil,
37
+ element_already_matched: false
38
+ )
39
+ config.extract(params)
40
+ end
41
+ end
42
+ end
data/purehtml.gemspec ADDED
@@ -0,0 +1,29 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "lib/purehtml/version"
4
+
5
+ Gem::Specification.new do |spec|
6
+ spec.name = "purehtml"
7
+ spec.version = PureHTML::VERSION
8
+ spec.authors = ["Fatih Kaya"]
9
+ spec.email = ["1994274@gmail.com"]
10
+
11
+ spec.summary = "Declarative HTML parsing and data extraction library for Ruby"
12
+ spec.description = "PureHTML is a parsing specification and library for extracting JSON from HTML documents using simple YAML schemas."
13
+ spec.homepage = "https://purescraps.github.io/purehtml/"
14
+ spec.license = "MIT"
15
+ spec.required_ruby_version = ">= 2.6.0"
16
+
17
+ spec.metadata["homepage_uri"] = spec.homepage
18
+ spec.metadata["source_code_uri"] = "https://github.com/purescraps/purehtml"
19
+
20
+ spec.files = Dir.chdir(File.expand_path(__dir__)) do
21
+ Dir.glob("{lib}/**/*") + %w[README.md LICENSE purehtml.gemspec Rakefile]
22
+ end
23
+ spec.require_paths = ["lib"]
24
+
25
+ spec.add_dependency "nokogiri", ">= 1.10.0"
26
+
27
+ spec.add_development_dependency "rake", ">= 12.0"
28
+ spec.add_development_dependency "minitest", ">= 5.0"
29
+ end
metadata ADDED
@@ -0,0 +1,126 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: purehtml
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - Fatih Kaya
8
+ bindir: bin
9
+ cert_chain: []
10
+ date: 1980-01-02 00:00:00.000000000 Z
11
+ dependencies:
12
+ - !ruby/object:Gem::Dependency
13
+ name: nokogiri
14
+ requirement: !ruby/object:Gem::Requirement
15
+ requirements:
16
+ - - ">="
17
+ - !ruby/object:Gem::Version
18
+ version: 1.10.0
19
+ type: :runtime
20
+ prerelease: false
21
+ version_requirements: !ruby/object:Gem::Requirement
22
+ requirements:
23
+ - - ">="
24
+ - !ruby/object:Gem::Version
25
+ version: 1.10.0
26
+ - !ruby/object:Gem::Dependency
27
+ name: rake
28
+ requirement: !ruby/object:Gem::Requirement
29
+ requirements:
30
+ - - ">="
31
+ - !ruby/object:Gem::Version
32
+ version: '12.0'
33
+ type: :development
34
+ prerelease: false
35
+ version_requirements: !ruby/object:Gem::Requirement
36
+ requirements:
37
+ - - ">="
38
+ - !ruby/object:Gem::Version
39
+ version: '12.0'
40
+ - !ruby/object:Gem::Dependency
41
+ name: minitest
42
+ requirement: !ruby/object:Gem::Requirement
43
+ requirements:
44
+ - - ">="
45
+ - !ruby/object:Gem::Version
46
+ version: '5.0'
47
+ type: :development
48
+ prerelease: false
49
+ version_requirements: !ruby/object:Gem::Requirement
50
+ requirements:
51
+ - - ">="
52
+ - !ruby/object:Gem::Version
53
+ version: '5.0'
54
+ description: PureHTML is a parsing specification and library for extracting JSON from
55
+ HTML documents using simple YAML schemas.
56
+ email:
57
+ - 1994274@gmail.com
58
+ executables: []
59
+ extensions: []
60
+ extra_rdoc_files: []
61
+ files:
62
+ - LICENSE
63
+ - README.md
64
+ - Rakefile
65
+ - lib/purehtml.rb
66
+ - lib/purehtml/backend.rb
67
+ - lib/purehtml/config.rb
68
+ - lib/purehtml/config_factory.rb
69
+ - lib/purehtml/configs/array_config.rb
70
+ - lib/purehtml/configs/constant_config.rb
71
+ - lib/purehtml/configs/default_value_config.rb
72
+ - lib/purehtml/configs/object_config.rb
73
+ - lib/purehtml/configs/primitive_value_config.rb
74
+ - lib/purehtml/configs/union_config.rb
75
+ - lib/purehtml/configs/with_selector.rb
76
+ - lib/purehtml/errors.rb
77
+ - lib/purehtml/extract_params.rb
78
+ - lib/purehtml/transformer.rb
79
+ - lib/purehtml/transformer_factory.rb
80
+ - lib/purehtml/transformers.rb
81
+ - lib/purehtml/transformers/attr.rb
82
+ - lib/purehtml/transformers/boolean.rb
83
+ - lib/purehtml/transformers/capitalize.rb
84
+ - lib/purehtml/transformers/exists.rb
85
+ - lib/purehtml/transformers/html.rb
86
+ - lib/purehtml/transformers/join.rb
87
+ - lib/purehtml/transformers/json.rb
88
+ - lib/purehtml/transformers/jsonpath.rb
89
+ - lib/purehtml/transformers/length.rb
90
+ - lib/purehtml/transformers/lower.rb
91
+ - lib/purehtml/transformers/number.rb
92
+ - lib/purehtml/transformers/remove_last_path_section.rb
93
+ - lib/purehtml/transformers/remove_url_query_param.rb
94
+ - lib/purehtml/transformers/replace.rb
95
+ - lib/purehtml/transformers/resolve.rb
96
+ - lib/purehtml/transformers/split.rb
97
+ - lib/purehtml/transformers/text.rb
98
+ - lib/purehtml/transformers/trim.rb
99
+ - lib/purehtml/transformers/upper.rb
100
+ - lib/purehtml/transformers/url_query_param.rb
101
+ - lib/purehtml/version.rb
102
+ - purehtml.gemspec
103
+ homepage: https://purescraps.github.io/purehtml/
104
+ licenses:
105
+ - MIT
106
+ metadata:
107
+ homepage_uri: https://purescraps.github.io/purehtml/
108
+ source_code_uri: https://github.com/purescraps/purehtml
109
+ rdoc_options: []
110
+ require_paths:
111
+ - lib
112
+ required_ruby_version: !ruby/object:Gem::Requirement
113
+ requirements:
114
+ - - ">="
115
+ - !ruby/object:Gem::Version
116
+ version: 2.6.0
117
+ required_rubygems_version: !ruby/object:Gem::Requirement
118
+ requirements:
119
+ - - ">="
120
+ - !ruby/object:Gem::Version
121
+ version: '0'
122
+ requirements: []
123
+ rubygems_version: 4.0.20
124
+ specification_version: 4
125
+ summary: Declarative HTML parsing and data extraction library for Ruby
126
+ test_files: []