html2rss 0.0.1 → 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.changelogrc +19 -0
- data/.gitignore +1 -0
- data/.rubocop.yml +46 -0
- data/.travis.yml +13 -4
- data/CHANGELOG.md +115 -0
- data/Gemfile.lock +28 -20
- data/README.md +22 -3
- data/html2rss.gemspec +5 -4
- data/lib/html2rss/attribute_post_processors.rb +16 -0
- data/lib/html2rss/attribute_post_processors/parse_time.rb +13 -0
- data/lib/html2rss/attribute_post_processors/parse_uri.rb +13 -0
- data/lib/html2rss/attribute_post_processors/sanitize_html.rb +23 -0
- data/lib/html2rss/attribute_post_processors/substring.rb +15 -0
- data/lib/html2rss/attribute_post_processors/template.rb +33 -0
- data/lib/html2rss/config.rb +3 -3
- data/lib/html2rss/feed_builder.rb +5 -3
- data/lib/html2rss/item.rb +24 -12
- data/lib/html2rss/item_extractor.rb +16 -16
- data/lib/html2rss/version.rb +1 -1
- data/support/changelog.md +15 -0
- data/support/logo.png +0 -0
- metadata +45 -20
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 1cc71ee2722757d0f4cceaf8aea1b9007496acf35fdf4da37a5dea998233258d
|
|
4
|
+
data.tar.gz: '0009a02fe85a6e2b088aec00944e1dc46233ce6878b8558782286be0e020511b'
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 2de4befd1f3beec2ee094f2c792ec94a1bde37c3da1f30eb737403ae04eddb36b54a43b86163f8b348229c42a5d38aae0bc7657c677bdbe95a54bee26b1fa6d9
|
|
7
|
+
data.tar.gz: f344bb3046c0493d9979e9daf53d5f3725798c96093dbbff6164a1339fd80bf54ad60868eb55dbc5a67239584f825cb0f5f6ca18c4e3e7f8f67b6698d26f3adf
|
data/.changelogrc
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
{
|
|
2
|
+
"app_name": "html2rss",
|
|
3
|
+
"logo": "https://github.com/gildesmarais/html2rss/raw/master/support/logo.png",
|
|
4
|
+
"intro": "Generate RSS feeds by scraping websites by providing a config.",
|
|
5
|
+
"debug": "true",
|
|
6
|
+
"template": "support/changelog.md",
|
|
7
|
+
"sections": [
|
|
8
|
+
{ "title": "Bugfixes", "grep": "^fix" },
|
|
9
|
+
{ "title": "Features", "grep": "^feat" },
|
|
10
|
+
{ "title": "Documentation", "grep": "^docs" },
|
|
11
|
+
{ "title": "Breaking changes", "grep": "BREAKING" },
|
|
12
|
+
{ "title": "Refactorings", "grep": "^refactor" },
|
|
13
|
+
{ "title": "Code style", "grep": "^style" },
|
|
14
|
+
{ "title": "Test", "grep": "^spec" },
|
|
15
|
+
{ "title": "Chore", "grep": "^chore" },
|
|
16
|
+
{ "title": "Branches merged", "grep": "^Merge branch" },
|
|
17
|
+
{ "title": "Pull requests merged", "grep": "^Merge pull request" }
|
|
18
|
+
]
|
|
19
|
+
}
|
data/.gitignore
CHANGED
data/.rubocop.yml
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
AllCops:
|
|
2
|
+
TargetRubyVersion: 2.4
|
|
3
|
+
DisplayCopNames: true
|
|
4
|
+
|
|
5
|
+
Metrics/LineLength:
|
|
6
|
+
Max: 110
|
|
7
|
+
|
|
8
|
+
Metrics/BlockLength:
|
|
9
|
+
Exclude:
|
|
10
|
+
- "**/*_spec.rb"
|
|
11
|
+
- html2rss.gemspec
|
|
12
|
+
|
|
13
|
+
Metrics/ModuleLength:
|
|
14
|
+
Exclude:
|
|
15
|
+
- "**/*_spec.rb"
|
|
16
|
+
|
|
17
|
+
Documentation:
|
|
18
|
+
Enabled: false
|
|
19
|
+
|
|
20
|
+
Style/BlockDelimiters:
|
|
21
|
+
Enabled: false
|
|
22
|
+
|
|
23
|
+
Style/FrozenStringLiteralComment:
|
|
24
|
+
Enabled: false
|
|
25
|
+
|
|
26
|
+
Style/ParallelAssignment:
|
|
27
|
+
Enabled: false
|
|
28
|
+
|
|
29
|
+
Style/AsciiComments:
|
|
30
|
+
Enabled: false
|
|
31
|
+
|
|
32
|
+
Style/BracesAroundHashParameters:
|
|
33
|
+
Description: 'Enforce braces style around hash parameters.'
|
|
34
|
+
Enabled: true
|
|
35
|
+
|
|
36
|
+
Style/HashSyntax:
|
|
37
|
+
Description: >-
|
|
38
|
+
Prefer Ruby 1.9 hash syntax { a: 1, b: 2 } over 1.8 syntax
|
|
39
|
+
{ :a => 1, :b => 2 }.
|
|
40
|
+
StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#hash-literals'
|
|
41
|
+
Enabled: true
|
|
42
|
+
|
|
43
|
+
Layout/SpaceInsideParens:
|
|
44
|
+
Description: 'No spaces after ( or before ).'
|
|
45
|
+
StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#no-spaces-braces'
|
|
46
|
+
Enabled: true
|
data/.travis.yml
CHANGED
|
@@ -1,15 +1,24 @@
|
|
|
1
1
|
sudo: false
|
|
2
2
|
language: ruby
|
|
3
|
+
cache: bundler
|
|
3
4
|
|
|
4
5
|
before_install:
|
|
5
6
|
- gem update --system
|
|
6
7
|
- gem install bundler
|
|
7
|
-
|
|
8
|
+
|
|
9
|
+
bundler_args: "--jobs=3 --retry=3"
|
|
8
10
|
|
|
9
11
|
rvm:
|
|
10
|
-
- 2.3.
|
|
11
|
-
- 2.4.
|
|
12
|
-
- 2.5.
|
|
12
|
+
- 2.3.8
|
|
13
|
+
- 2.4.5
|
|
14
|
+
- 2.5.3
|
|
13
15
|
|
|
14
16
|
script:
|
|
15
17
|
- bundle exec rspec
|
|
18
|
+
|
|
19
|
+
deploy:
|
|
20
|
+
provider: rubygems
|
|
21
|
+
api_key:
|
|
22
|
+
secure: bM3Yl8iWdB1Amra3Bm6bIH/mTwHcRhZrX8etFFbJANxIbkhzUOyTKcDMYiWUVM/mBzzv0NOuRejrDR6R0v7E2udrKcLQFCBtv7HqPAXIlkEEyxZy+M1kTqcPzP872E+ZKTn93vCzbiXBLYoMmqgCzqvcO87IBYNzTURHkfFjaYJJdVyZ5EVtbpXf4FhBvuQf9LTk/ocClgwYeuqd+45lO7qHoPatsvbY0vCOfKaiwkdOkBt+hjc56awcYSc9CXn0DCatebPQmQmdrqFd8fKgyCatWS3n+8TPmvzVfNJe44wg3oNfHbWruP85I2LE9ei1iG+iGQIF60fMhGgMJ4EM3REXDE5Mg+GA5uJcgH9Poirut3Ih65jtAzYNGohlmEmc7ysKc0dmG1O3ndwrHjh5KePrOAGDaW6QKG+m5ebIZ+mgrEA+ZVU1mjDM8FlbSKAayoPloslZdllSv7miwGzh6xrHWGQSCURZAkygFh+Kd+Kg1eVlEs+n6aObod82mEOfBPvWPacOrE2fY4B0ocFOKotZBCZSD0ZIixlyslRTnmcJfpRNlYLsQ56oy5uPNUccPQ86NSmmE+qbRdPCLQCKLPm2iYBgOa5iQrfHR/fUgcO0skAZiW4o9QflDgIFS/G+BE6FMHIvjkKA6Ae4KbqGzlF5pGFdo6p4MhlvubwjsVI=
|
|
23
|
+
on:
|
|
24
|
+
tags: true
|
data/CHANGELOG.md
ADDED
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
<img width="300px" src="https://github.com/gildesmarais/html2rss/raw/master/support/logo.png" />
|
|
2
|
+
|
|
3
|
+
# html2rss
|
|
4
|
+
|
|
5
|
+
_Generate RSS feeds by scraping websites by providing a config._
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
## Bugfixes
|
|
10
|
+
- handling of url query breaks processing
|
|
11
|
+
([ace289e9](git@github.com:gildesmarais/html2rss/commit/ace289e911b69cb92433cac6f1ca0403715d8286))
|
|
12
|
+
- only set supported attributes on rss item
|
|
13
|
+
([dae0d8e7](git@github.com:gildesmarais/html2rss/commit/dae0d8e75541e810275e789a23971a61e60a2154))
|
|
14
|
+
|
|
15
|
+
- **config**
|
|
16
|
+
- feed generation fails
|
|
17
|
+
([7dd55869](git@github.com:gildesmarais/html2rss/commit/7dd55869f79b1de76c004bf0e82d13b16b5b3f0d))
|
|
18
|
+
|
|
19
|
+
- **parse_uri**
|
|
20
|
+
- handle non-absolute paths
|
|
21
|
+
([92150257](git@github.com:gildesmarais/html2rss/commit/921502574e4436d65a30e1d34b9b31f238336247))
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
## Features
|
|
27
|
+
- add logo [skip ci]
|
|
28
|
+
([857a55fd](git@github.com:gildesmarais/html2rss/commit/857a55fd8c932930d96c47c5abe57f0507356df1))
|
|
29
|
+
- require updated to be present
|
|
30
|
+
([e1bedaec](git@github.com:gildesmarais/html2rss/commit/e1bedaecc91e874fe24e96000612abb9cd11e9fe))
|
|
31
|
+
- do not fail on invalid item, just skip it
|
|
32
|
+
([3b83d715](git@github.com:gildesmarais/html2rss/commit/3b83d715619abbc33b124de1945d17cb0dc7edb0))
|
|
33
|
+
|
|
34
|
+
- **item_extractor**
|
|
35
|
+
- text strips strings
|
|
36
|
+
([f5982859](git@github.com:gildesmarais/html2rss/commit/f59828593dca663bdbe8699392594e2d18658f8f))
|
|
37
|
+
- add static and current_time
|
|
38
|
+
([25043dcb](git@github.com:gildesmarais/html2rss/commit/25043dcbd8f0f4901202f4a2f66b355ac48825a8))
|
|
39
|
+
- handle absolute urls
|
|
40
|
+
([f96be008](git@github.com:gildesmarais/html2rss/commit/f96be00857bdcded02d52dd62ec22b9b52c803ed))
|
|
41
|
+
|
|
42
|
+
- **post_processing**
|
|
43
|
+
- add configurable post_processing (#5)
|
|
44
|
+
([4cf6caca](git@github.com:gildesmarais/html2rss/commit/4cf6cacac00bd3c0c53d584ca11274ba24b03ef7),
|
|
45
|
+
[#1](git@github.com:gildesmarais/html2rss/issues/1))
|
|
46
|
+
|
|
47
|
+
- **post_processor**
|
|
48
|
+
- add substring
|
|
49
|
+
([6f2a32a6](git@github.com:gildesmarais/html2rss/commit/6f2a32a6304ef9956577711173de681daf93f55f))
|
|
50
|
+
|
|
51
|
+
- **postprocessors**
|
|
52
|
+
- add Template (#6)
|
|
53
|
+
([f1db542e](git@github.com:gildesmarais/html2rss/commit/f1db542e8c1e9e09a066a3cd6c8514a6ca0aa871),
|
|
54
|
+
[#4](git@github.com:gildesmarais/html2rss/issues/4))
|
|
55
|
+
|
|
56
|
+
- **sanitize_html**
|
|
57
|
+
- add target="_blank" to anchors
|
|
58
|
+
([975a73bf](git@github.com:gildesmarais/html2rss/commit/975a73bfd396ba5942bc0ea80eebd14cc37ad776))
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
## Documentation
|
|
64
|
+
- create the changelog
|
|
65
|
+
([5c561db5](git@github.com:gildesmarais/html2rss/commit/5c561db51d4e0b8592b1c82812ab5cdbe9320b70))
|
|
66
|
+
- add tips and tricks
|
|
67
|
+
([ea978240](git@github.com:gildesmarais/html2rss/commit/ea9782408107f3637a4c9665396f511fc07be19b))
|
|
68
|
+
- update readme
|
|
69
|
+
([4743167c](git@github.com:gildesmarais/html2rss/commit/4743167c86959e83524ffb7282c562413a651797))
|
|
70
|
+
- add note about html2rss-web
|
|
71
|
+
([3371c12f](git@github.com:gildesmarais/html2rss/commit/3371c12ffc6c8d3c29073d03ff206886a39401cd))
|
|
72
|
+
- add a badge for travis-ci
|
|
73
|
+
([8818d4f4](git@github.com:gildesmarais/html2rss/commit/8818d4f464a9c163ebc9665d01719e2bab132bd6))
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
## Test
|
|
79
|
+
- don't be so lazy when matching strings
|
|
80
|
+
([6a0eb627](git@github.com:gildesmarais/html2rss/commit/6a0eb62765523a1405fd269466b2fc57794eac7a))
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
## Chore
|
|
86
|
+
- upgrade sanitze gem to version 5.0.0
|
|
87
|
+
([8c4bf3a4](git@github.com:gildesmarais/html2rss/commit/8c4bf3a44885758e395568ec452a7cffdb9a0389))
|
|
88
|
+
- rubocop autocorrect
|
|
89
|
+
([af6b9fac](git@github.com:gildesmarais/html2rss/commit/af6b9facca547d3ca3ce9ef0d1227707cd16eaea))
|
|
90
|
+
- build against latest ruby releases
|
|
91
|
+
([17ba79ac](git@github.com:gildesmarais/html2rss/commit/17ba79acd2f68da1fcc984368d3e6de3437cbf1b))
|
|
92
|
+
- update dependencies
|
|
93
|
+
([855279b4](git@github.com:gildesmarais/html2rss/commit/855279b46d584a8a8c2a317529f7a4be550eaf15))
|
|
94
|
+
- update dependencies
|
|
95
|
+
([46e5a283](git@github.com:gildesmarais/html2rss/commit/46e5a2832d1f2fe1353dcc2a8d82a9786f15f6bd))
|
|
96
|
+
- add simplecov
|
|
97
|
+
([b4e1144b](git@github.com:gildesmarais/html2rss/commit/b4e1144b7f8f90126e528cc4a4ec048113d93634))
|
|
98
|
+
|
|
99
|
+
- **changelog**
|
|
100
|
+
- add generation with git-changelog
|
|
101
|
+
([07ad5a51](git@github.com:gildesmarais/html2rss/commit/07ad5a513f0951ee988426abda4b8c233411ead7))
|
|
102
|
+
|
|
103
|
+
- **travis**
|
|
104
|
+
- use cache for bundler
|
|
105
|
+
([ac76b3b2](git@github.com:gildesmarais/html2rss/commit/ac76b3b2dd94adecd4927de18651800438a7e7ba))
|
|
106
|
+
- setup autorelease to rubygems
|
|
107
|
+
([eb9c8e1b](git@github.com:gildesmarais/html2rss/commit/eb9c8e1b16902dc0e174a0cccb6eb9227307ce82))
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
---
|
|
114
|
+
<sub><sup>*Generated with [git-changelog](https://github.com/rafinskipg/git-changelog). If you have any problems or suggestions, create an issue.* :) **Thanks** </sub></sup>
|
|
115
|
+
|
data/Gemfile.lock
CHANGED
|
@@ -4,7 +4,7 @@ PATH
|
|
|
4
4
|
html2rss (0.0.1)
|
|
5
5
|
faraday (~> 0.15)
|
|
6
6
|
nokogiri (~> 1.8)
|
|
7
|
-
sanitize (~>
|
|
7
|
+
sanitize (~> 5.0)
|
|
8
8
|
|
|
9
9
|
GEM
|
|
10
10
|
remote: https://rubygems.org/
|
|
@@ -12,31 +12,38 @@ GEM
|
|
|
12
12
|
byebug (10.0.2)
|
|
13
13
|
crass (1.0.4)
|
|
14
14
|
diff-lcs (1.3)
|
|
15
|
-
|
|
15
|
+
docile (1.3.1)
|
|
16
|
+
faraday (0.15.3)
|
|
16
17
|
multipart-post (>= 1.2, < 3)
|
|
18
|
+
json (2.1.0)
|
|
17
19
|
mini_portile2 (2.3.0)
|
|
18
20
|
multipart-post (2.0.0)
|
|
19
|
-
nokogiri (1.8.
|
|
21
|
+
nokogiri (1.8.5)
|
|
20
22
|
mini_portile2 (~> 2.3.0)
|
|
21
|
-
nokogumbo (
|
|
22
|
-
nokogiri
|
|
23
|
-
rspec (3.
|
|
24
|
-
rspec-core (~> 3.
|
|
25
|
-
rspec-expectations (~> 3.
|
|
26
|
-
rspec-mocks (~> 3.
|
|
27
|
-
rspec-core (3.
|
|
28
|
-
rspec-support (~> 3.
|
|
29
|
-
rspec-expectations (3.
|
|
23
|
+
nokogumbo (2.0.0)
|
|
24
|
+
nokogiri (~> 1.8, >= 1.8.4)
|
|
25
|
+
rspec (3.8.0)
|
|
26
|
+
rspec-core (~> 3.8.0)
|
|
27
|
+
rspec-expectations (~> 3.8.0)
|
|
28
|
+
rspec-mocks (~> 3.8.0)
|
|
29
|
+
rspec-core (3.8.0)
|
|
30
|
+
rspec-support (~> 3.8.0)
|
|
31
|
+
rspec-expectations (3.8.2)
|
|
30
32
|
diff-lcs (>= 1.2.0, < 2.0)
|
|
31
|
-
rspec-support (~> 3.
|
|
32
|
-
rspec-mocks (3.
|
|
33
|
+
rspec-support (~> 3.8.0)
|
|
34
|
+
rspec-mocks (3.8.0)
|
|
33
35
|
diff-lcs (>= 1.2.0, < 2.0)
|
|
34
|
-
rspec-support (~> 3.
|
|
35
|
-
rspec-support (3.
|
|
36
|
-
sanitize (
|
|
36
|
+
rspec-support (~> 3.8.0)
|
|
37
|
+
rspec-support (3.8.0)
|
|
38
|
+
sanitize (5.0.0)
|
|
37
39
|
crass (~> 1.0.2)
|
|
38
|
-
nokogiri (>= 1.
|
|
39
|
-
nokogumbo (~>
|
|
40
|
+
nokogiri (>= 1.8.0)
|
|
41
|
+
nokogumbo (~> 2.0)
|
|
42
|
+
simplecov (0.16.1)
|
|
43
|
+
docile (~> 1.1)
|
|
44
|
+
json (>= 1.8, < 3)
|
|
45
|
+
simplecov-html (~> 0.10.0)
|
|
46
|
+
simplecov-html (0.10.2)
|
|
40
47
|
vcr (4.0.0)
|
|
41
48
|
|
|
42
49
|
PLATFORMS
|
|
@@ -47,7 +54,8 @@ DEPENDENCIES
|
|
|
47
54
|
byebug (~> 10.0)
|
|
48
55
|
html2rss!
|
|
49
56
|
rspec (~> 3.0)
|
|
57
|
+
simplecov
|
|
50
58
|
vcr (~> 4.0)
|
|
51
59
|
|
|
52
60
|
BUNDLED WITH
|
|
53
|
-
1.16.
|
|
61
|
+
1.16.6
|
data/README.md
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
|
-
|
|
1
|
+
|
|
2
|
+
|
|
3
|
+
# html2rss [](https://travis-ci.org/gildesmarais/html2rss)
|
|
2
4
|
|
|
3
5
|
Request and convert an HTML document to an RSS feed via a config object.
|
|
4
6
|
The config contains the URL to scrape and the selectors needed to extract
|
|
@@ -24,14 +26,26 @@ Or install it yourself as:
|
|
|
24
26
|
|
|
25
27
|
$ gem install html2rss
|
|
26
28
|
|
|
27
|
-
## Usage
|
|
29
|
+
## Usage
|
|
30
|
+
|
|
31
|
+
## Usage with a YAML file
|
|
28
32
|
|
|
29
|
-
Create a YAML config file. Find an example at `rspec/config.test.yml
|
|
33
|
+
Create a YAML config file. Find an example at [`rspec/config.test.yml`](https://github.com/gildesmarais/html2rss/blob/master/spec/config.test.yml).
|
|
30
34
|
|
|
31
35
|
`Html2rss.feed_from_yaml_config(File.join(['spec', 'config.test.yml']), 'nuxt-releases')` returns
|
|
32
36
|
|
|
33
37
|
an `RSS:Rss` object.
|
|
34
38
|
|
|
39
|
+
## Usage in a web application
|
|
40
|
+
|
|
41
|
+
Find a minimal Sintra app which exposes your feeds to HTTP endpoints here:
|
|
42
|
+
[gildesmarais/html2rss-web](https://github.com/gildesmarais/html2rss-web)
|
|
43
|
+
|
|
44
|
+
### Tips and tricks
|
|
45
|
+
|
|
46
|
+
- Check that the channel url does not redirect to a mobile page
|
|
47
|
+
- fiddling with [`curl`](https://github.com/curl/curl) and [`pup`](https://github.com/ericchiang/pup) to find the selectors seems quite efficient
|
|
48
|
+
|
|
35
49
|
## Development
|
|
36
50
|
|
|
37
51
|
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
|
@@ -40,6 +54,11 @@ After checking out the repo, run `bin/setup` to install dependencies. Then, run
|
|
|
40
54
|
|
|
41
55
|
Bug reports and pull requests are welcome on GitHub at https://github.com/gildesmarais/html2rss.
|
|
42
56
|
|
|
57
|
+
## Changelog generation
|
|
58
|
+
|
|
59
|
+
The `CHANGELOG.md` can be generated automatically.
|
|
60
|
+
Install [git-changelog](https://www.npmjs.com/package/git-changelog) globally and run `git-changelog` afterwards.
|
|
61
|
+
|
|
43
62
|
## License
|
|
44
63
|
|
|
45
64
|
The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
|
data/html2rss.gemspec
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
lib = File.expand_path('
|
|
1
|
+
lib = File.expand_path('lib', __dir__)
|
|
2
2
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
3
3
|
require 'html2rss/version'
|
|
4
4
|
|
|
@@ -28,11 +28,12 @@ Gem::Specification.new do |spec|
|
|
|
28
28
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
|
29
29
|
spec.require_paths = ['lib']
|
|
30
30
|
|
|
31
|
-
spec.add_dependency 'nokogiri', '~> 1.8'
|
|
32
|
-
spec.add_dependency 'sanitize', '~> 4.6'
|
|
33
31
|
spec.add_dependency 'faraday', '~> 0.15'
|
|
32
|
+
spec.add_dependency 'nokogiri', '~> 1.8'
|
|
33
|
+
spec.add_dependency 'sanitize', '~> 5.0'
|
|
34
34
|
spec.add_development_dependency 'bundler', '~> 1.16'
|
|
35
|
+
spec.add_development_dependency 'byebug', '~> 10.0'
|
|
35
36
|
spec.add_development_dependency 'rspec', '~> 3.0'
|
|
37
|
+
spec.add_development_dependency 'simplecov'
|
|
36
38
|
spec.add_development_dependency 'vcr', '~> 4.0'
|
|
37
|
-
spec.add_development_dependency 'byebug', '~> 10.0'
|
|
38
39
|
end
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
require_relative 'attribute_post_processors/parse_time'
|
|
2
|
+
require_relative 'attribute_post_processors/parse_uri'
|
|
3
|
+
require_relative 'attribute_post_processors/sanitize_html'
|
|
4
|
+
require_relative 'attribute_post_processors/substring'
|
|
5
|
+
require_relative 'attribute_post_processors/template'
|
|
6
|
+
|
|
7
|
+
module Html2rss
|
|
8
|
+
module AttributePostProcessors
|
|
9
|
+
def self.get_processor(options)
|
|
10
|
+
camel_cased_option = options['name'].split('_').collect(&:capitalize).join
|
|
11
|
+
class_name = ['Html2rss', 'AttributePostProcessors', camel_cased_option].join('::')
|
|
12
|
+
|
|
13
|
+
Object.const_get(class_name)
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
end
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
require 'sanitize'
|
|
2
|
+
|
|
3
|
+
module Html2rss
|
|
4
|
+
module AttributePostProcessors
|
|
5
|
+
class SanitizeHtml
|
|
6
|
+
def initialize(value, _options, _item)
|
|
7
|
+
@value = value
|
|
8
|
+
end
|
|
9
|
+
|
|
10
|
+
def get
|
|
11
|
+
Sanitize.fragment(@value, Sanitize::Config.merge(
|
|
12
|
+
Sanitize::Config::RELAXED,
|
|
13
|
+
add_attributes: {
|
|
14
|
+
'a' => {
|
|
15
|
+
'rel' => 'nofollow noopener noreferrer',
|
|
16
|
+
'target' => '_blank'
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
))
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
module Html2rss
|
|
2
|
+
module AttributePostProcessors
|
|
3
|
+
class Substring
|
|
4
|
+
def initialize(value, options, _item)
|
|
5
|
+
@value = value
|
|
6
|
+
@options = options
|
|
7
|
+
end
|
|
8
|
+
|
|
9
|
+
def get
|
|
10
|
+
ending = @options['end'].to_i.positive? ? @options['end'].to_i : @value.length
|
|
11
|
+
@value[@options['start'].to_i..ending]
|
|
12
|
+
end
|
|
13
|
+
end
|
|
14
|
+
end
|
|
15
|
+
end
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
require 'sanitize'
|
|
2
|
+
|
|
3
|
+
module Html2rss
|
|
4
|
+
module AttributePostProcessors
|
|
5
|
+
class Template
|
|
6
|
+
def initialize(value, options, item)
|
|
7
|
+
@value = value
|
|
8
|
+
@options = options
|
|
9
|
+
@item = item
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
def get
|
|
13
|
+
string % methods
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
private
|
|
17
|
+
|
|
18
|
+
def string
|
|
19
|
+
@options['string']
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def methods
|
|
23
|
+
@methods ||= @options['methods'].map { |method|
|
|
24
|
+
if method == 'self'
|
|
25
|
+
@value
|
|
26
|
+
else
|
|
27
|
+
@item.send(method.to_sym)&.to_s
|
|
28
|
+
end
|
|
29
|
+
}
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|
data/lib/html2rss/config.rb
CHANGED
|
@@ -38,7 +38,7 @@ module Html2rss
|
|
|
38
38
|
end
|
|
39
39
|
|
|
40
40
|
def options(name)
|
|
41
|
-
feed_config.dig('selectors',
|
|
41
|
+
feed_config.dig('selectors').fetch(name, {}).merge('channel' => channel_config)
|
|
42
42
|
end
|
|
43
43
|
|
|
44
44
|
def selector(name)
|
|
@@ -46,8 +46,8 @@ module Html2rss
|
|
|
46
46
|
end
|
|
47
47
|
|
|
48
48
|
def attribute_names
|
|
49
|
-
attribute_names = feed_config.fetch('selectors', {}).keys.map(&:
|
|
50
|
-
attribute_names.delete(
|
|
49
|
+
attribute_names = feed_config.fetch('selectors', {}).keys.map(&:to_s)
|
|
50
|
+
attribute_names.delete('items')
|
|
51
51
|
attribute_names
|
|
52
52
|
end
|
|
53
53
|
end
|
|
@@ -22,7 +22,7 @@ module Html2rss
|
|
|
22
22
|
private
|
|
23
23
|
|
|
24
24
|
def add_channel_to_maker(maker)
|
|
25
|
-
[
|
|
25
|
+
%i[language author title description link ttl].each do |attribute_name|
|
|
26
26
|
maker.channel.send("#{attribute_name}=".to_sym, config.send(attribute_name))
|
|
27
27
|
end
|
|
28
28
|
|
|
@@ -31,12 +31,14 @@ module Html2rss
|
|
|
31
31
|
end
|
|
32
32
|
|
|
33
33
|
def feed_items
|
|
34
|
-
Item.from_url config.url, config
|
|
34
|
+
@feed_items ||= Item.from_url config.url, config
|
|
35
35
|
end
|
|
36
36
|
|
|
37
37
|
def add_item_to_items(feed_item, items)
|
|
38
|
+
return unless feed_item.valid?
|
|
39
|
+
|
|
38
40
|
items.new_item do |rss_item|
|
|
39
|
-
|
|
41
|
+
feed_item.available_attributes.each do |attribute_name|
|
|
40
42
|
rss_item.send("#{attribute_name}=".to_sym, feed_item.send(attribute_name))
|
|
41
43
|
|
|
42
44
|
rss_item.guid.content = Digest::SHA1.hexdigest(feed_item.title)
|
data/lib/html2rss/item.rb
CHANGED
|
@@ -2,6 +2,7 @@ require 'faraday'
|
|
|
2
2
|
require 'open-uri'
|
|
3
3
|
require 'nokogiri'
|
|
4
4
|
require_relative 'item_extractor'
|
|
5
|
+
require_relative 'attribute_post_processors'
|
|
5
6
|
|
|
6
7
|
module Html2rss
|
|
7
8
|
class Item
|
|
@@ -24,26 +25,37 @@ module Html2rss
|
|
|
24
25
|
proc = ItemExtractor.const_get extractor.upcase.to_sym
|
|
25
26
|
value = proc.call(xml, attribute_config)
|
|
26
27
|
|
|
27
|
-
post_process
|
|
28
|
+
post_process_options = attribute_config.fetch('post_process', false)
|
|
29
|
+
value = post_process(value, post_process_options) if post_process_options
|
|
30
|
+
|
|
31
|
+
value
|
|
28
32
|
end
|
|
29
33
|
|
|
30
|
-
def
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
34
|
+
def available_attributes
|
|
35
|
+
# TODO: support optional attributes, e.g. category, enclosure, source
|
|
36
|
+
@available_attributes ||= (%w[title link description author comments updated] & @config.attribute_names)
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def valid?
|
|
40
|
+
return false if [title.to_s, description.to_s].join('') == ''
|
|
41
|
+
|
|
42
|
+
true
|
|
39
43
|
end
|
|
40
44
|
|
|
41
45
|
def self.from_url(url, config)
|
|
42
46
|
connection = Faraday.new(url: url, headers: config.headers)
|
|
43
47
|
page = Nokogiri::HTML(connection.get.body)
|
|
44
|
-
page.css(config.selector('items')).map
|
|
48
|
+
page.css(config.selector('items')).map do |xml_item|
|
|
45
49
|
new xml_item, config
|
|
46
|
-
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
private
|
|
54
|
+
|
|
55
|
+
def post_process(value, options)
|
|
56
|
+
Html2rss::AttributePostProcessors.get_processor(options)
|
|
57
|
+
.new(value, options, self)
|
|
58
|
+
.get
|
|
47
59
|
end
|
|
48
60
|
end
|
|
49
61
|
end
|
|
@@ -1,25 +1,25 @@
|
|
|
1
|
-
require 'sanitize'
|
|
2
|
-
|
|
3
1
|
module Html2rss
|
|
4
2
|
module ItemExtractor
|
|
5
|
-
TEXT = proc { |xml, options| xml.css(options['selector'])&.text }
|
|
6
|
-
ATTRIBUTE = proc { |xml, options| xml.css(options['selector']).attr(options['attribute']) }
|
|
3
|
+
TEXT = proc { |xml, options| xml.css(options['selector'])&.text&.strip }
|
|
4
|
+
ATTRIBUTE = proc { |xml, options| xml.css(options['selector']).attr(options['attribute']).to_s }
|
|
7
5
|
|
|
8
6
|
HREF = proc { |xml, options|
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
uri
|
|
12
|
-
}
|
|
7
|
+
href = xml.css(options['selector']).attr('href').to_s
|
|
8
|
+
path, query = href.split('?')
|
|
13
9
|
|
|
14
|
-
|
|
15
|
-
|
|
10
|
+
if href.start_with?('http')
|
|
11
|
+
uri = URI(href)
|
|
12
|
+
else
|
|
13
|
+
uri = URI(options['channel']['url'])
|
|
14
|
+
uri.path = path.start_with?('/') ? path : "/#{path}"
|
|
15
|
+
uri.query = query
|
|
16
|
+
end
|
|
16
17
|
|
|
17
|
-
|
|
18
|
-
Sanitize::Config::RELAXED,
|
|
19
|
-
add_attributes: {
|
|
20
|
-
'a' => { 'rel' => 'nofollow noopener noreferrer' }
|
|
21
|
-
}
|
|
22
|
-
))
|
|
18
|
+
uri
|
|
23
19
|
}
|
|
20
|
+
|
|
21
|
+
HTML = proc { |xml, options| xml.css(options['selector']).to_s }
|
|
22
|
+
STATIC = proc { |_xml, options| options['static'] }
|
|
23
|
+
CURRENT_TIME = proc { |_xml, _options| Time.new }
|
|
24
24
|
end
|
|
25
25
|
end
|
data/lib/html2rss/version.rb
CHANGED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
<% if(logo) { %><img width="300px" src="<%= logo %>" /><%= '\n\n' %><% } %># <%= title %>
|
|
2
|
+
<% if(intro) { %><%= '\n' %>_<%= intro %>_<%= '\n' %><% } %>
|
|
3
|
+
<% if(version && (version.name || version.number)) { %>##<% if(version.name){%> <%= version.name %><% } %> <%= version.number %> <% if(version.date){ %>( <%= version.date %> )<% } %><%= '\n' %><% } %>
|
|
4
|
+
<% _.forEach(sections, function(section){
|
|
5
|
+
if(section.commitsCount > 0) { %>
|
|
6
|
+
## <%= section.title %>
|
|
7
|
+
<% _.forEach(section.commits, function(commit){ %> - <%= printCommit(commit, true) %><% }) %>
|
|
8
|
+
<% _.forEach(section.components, function(component){ %> - **<%= component.name %>**
|
|
9
|
+
<% _.forEach(component.commits, function(commit){ %> - <%= printCommit(commit, true) %><% }) %>
|
|
10
|
+
<% }) %>
|
|
11
|
+
<% } %>
|
|
12
|
+
<% }) %>
|
|
13
|
+
|
|
14
|
+
---
|
|
15
|
+
<sub><sup>*Generated with [git-changelog](https://github.com/rafinskipg/git-changelog). If you have any problems or suggestions, create an issue.* :) **Thanks** </sub></sup>
|
data/support/logo.png
ADDED
|
Binary file
|
metadata
CHANGED
|
@@ -1,57 +1,57 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: html2rss
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.0
|
|
4
|
+
version: 0.1.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Gil Desmarais
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2018-
|
|
11
|
+
date: 2018-11-04 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
|
-
name:
|
|
14
|
+
name: faraday
|
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
|
16
16
|
requirements:
|
|
17
17
|
- - "~>"
|
|
18
18
|
- !ruby/object:Gem::Version
|
|
19
|
-
version: '
|
|
19
|
+
version: '0.15'
|
|
20
20
|
type: :runtime
|
|
21
21
|
prerelease: false
|
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
|
23
23
|
requirements:
|
|
24
24
|
- - "~>"
|
|
25
25
|
- !ruby/object:Gem::Version
|
|
26
|
-
version: '
|
|
26
|
+
version: '0.15'
|
|
27
27
|
- !ruby/object:Gem::Dependency
|
|
28
|
-
name:
|
|
28
|
+
name: nokogiri
|
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
|
30
30
|
requirements:
|
|
31
31
|
- - "~>"
|
|
32
32
|
- !ruby/object:Gem::Version
|
|
33
|
-
version: '
|
|
33
|
+
version: '1.8'
|
|
34
34
|
type: :runtime
|
|
35
35
|
prerelease: false
|
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
|
37
37
|
requirements:
|
|
38
38
|
- - "~>"
|
|
39
39
|
- !ruby/object:Gem::Version
|
|
40
|
-
version: '
|
|
40
|
+
version: '1.8'
|
|
41
41
|
- !ruby/object:Gem::Dependency
|
|
42
|
-
name:
|
|
42
|
+
name: sanitize
|
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
|
44
44
|
requirements:
|
|
45
45
|
- - "~>"
|
|
46
46
|
- !ruby/object:Gem::Version
|
|
47
|
-
version: '0
|
|
47
|
+
version: '5.0'
|
|
48
48
|
type: :runtime
|
|
49
49
|
prerelease: false
|
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
|
51
51
|
requirements:
|
|
52
52
|
- - "~>"
|
|
53
53
|
- !ruby/object:Gem::Version
|
|
54
|
-
version: '0
|
|
54
|
+
version: '5.0'
|
|
55
55
|
- !ruby/object:Gem::Dependency
|
|
56
56
|
name: bundler
|
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -66,6 +66,20 @@ dependencies:
|
|
|
66
66
|
- - "~>"
|
|
67
67
|
- !ruby/object:Gem::Version
|
|
68
68
|
version: '1.16'
|
|
69
|
+
- !ruby/object:Gem::Dependency
|
|
70
|
+
name: byebug
|
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
|
72
|
+
requirements:
|
|
73
|
+
- - "~>"
|
|
74
|
+
- !ruby/object:Gem::Version
|
|
75
|
+
version: '10.0'
|
|
76
|
+
type: :development
|
|
77
|
+
prerelease: false
|
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
79
|
+
requirements:
|
|
80
|
+
- - "~>"
|
|
81
|
+
- !ruby/object:Gem::Version
|
|
82
|
+
version: '10.0'
|
|
69
83
|
- !ruby/object:Gem::Dependency
|
|
70
84
|
name: rspec
|
|
71
85
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -81,33 +95,33 @@ dependencies:
|
|
|
81
95
|
- !ruby/object:Gem::Version
|
|
82
96
|
version: '3.0'
|
|
83
97
|
- !ruby/object:Gem::Dependency
|
|
84
|
-
name:
|
|
98
|
+
name: simplecov
|
|
85
99
|
requirement: !ruby/object:Gem::Requirement
|
|
86
100
|
requirements:
|
|
87
|
-
- - "
|
|
101
|
+
- - ">="
|
|
88
102
|
- !ruby/object:Gem::Version
|
|
89
|
-
version: '
|
|
103
|
+
version: '0'
|
|
90
104
|
type: :development
|
|
91
105
|
prerelease: false
|
|
92
106
|
version_requirements: !ruby/object:Gem::Requirement
|
|
93
107
|
requirements:
|
|
94
|
-
- - "
|
|
108
|
+
- - ">="
|
|
95
109
|
- !ruby/object:Gem::Version
|
|
96
|
-
version: '
|
|
110
|
+
version: '0'
|
|
97
111
|
- !ruby/object:Gem::Dependency
|
|
98
|
-
name:
|
|
112
|
+
name: vcr
|
|
99
113
|
requirement: !ruby/object:Gem::Requirement
|
|
100
114
|
requirements:
|
|
101
115
|
- - "~>"
|
|
102
116
|
- !ruby/object:Gem::Version
|
|
103
|
-
version: '
|
|
117
|
+
version: '4.0'
|
|
104
118
|
type: :development
|
|
105
119
|
prerelease: false
|
|
106
120
|
version_requirements: !ruby/object:Gem::Requirement
|
|
107
121
|
requirements:
|
|
108
122
|
- - "~>"
|
|
109
123
|
- !ruby/object:Gem::Version
|
|
110
|
-
version: '
|
|
124
|
+
version: '4.0'
|
|
111
125
|
description: |-
|
|
112
126
|
Create your config object, include the url to scrape,
|
|
113
127
|
some selectors and get a RSS2 feed in return.
|
|
@@ -117,9 +131,12 @@ executables: []
|
|
|
117
131
|
extensions: []
|
|
118
132
|
extra_rdoc_files: []
|
|
119
133
|
files:
|
|
134
|
+
- ".changelogrc"
|
|
120
135
|
- ".gitignore"
|
|
121
136
|
- ".rspec"
|
|
137
|
+
- ".rubocop.yml"
|
|
122
138
|
- ".travis.yml"
|
|
139
|
+
- CHANGELOG.md
|
|
123
140
|
- Gemfile
|
|
124
141
|
- Gemfile.lock
|
|
125
142
|
- LICENSE
|
|
@@ -128,11 +145,19 @@ files:
|
|
|
128
145
|
- bin/setup
|
|
129
146
|
- html2rss.gemspec
|
|
130
147
|
- lib/html2rss.rb
|
|
148
|
+
- lib/html2rss/attribute_post_processors.rb
|
|
149
|
+
- lib/html2rss/attribute_post_processors/parse_time.rb
|
|
150
|
+
- lib/html2rss/attribute_post_processors/parse_uri.rb
|
|
151
|
+
- lib/html2rss/attribute_post_processors/sanitize_html.rb
|
|
152
|
+
- lib/html2rss/attribute_post_processors/substring.rb
|
|
153
|
+
- lib/html2rss/attribute_post_processors/template.rb
|
|
131
154
|
- lib/html2rss/config.rb
|
|
132
155
|
- lib/html2rss/feed_builder.rb
|
|
133
156
|
- lib/html2rss/item.rb
|
|
134
157
|
- lib/html2rss/item_extractor.rb
|
|
135
158
|
- lib/html2rss/version.rb
|
|
159
|
+
- support/changelog.md
|
|
160
|
+
- support/logo.png
|
|
136
161
|
homepage: https://github.com/gildesmarais/html2rss
|
|
137
162
|
licenses:
|
|
138
163
|
- MIT
|
|
@@ -154,7 +179,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
154
179
|
version: '0'
|
|
155
180
|
requirements: []
|
|
156
181
|
rubyforge_project:
|
|
157
|
-
rubygems_version: 2.7.
|
|
182
|
+
rubygems_version: 2.7.8
|
|
158
183
|
signing_key:
|
|
159
184
|
specification_version: 4
|
|
160
185
|
summary: Generate RSS feeds by scraping websites by providing a config.
|