offlinerss 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +11 -0
- data/Gemfile +5 -0
- data/Gemfile.lock +21 -0
- data/LICENSE +21 -0
- data/README.md +52 -0
- data/Rakefile +1 -0
- data/exe/offlinerss +52 -0
- data/offlinerss.gemspec +27 -0
- metadata +68 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: db62105a5c4a87c5f63c8b62a83ec0ca4e9e29de1b912197656c0151204e6545
|
4
|
+
data.tar.gz: 4fc5c9f5af64924987c0dd089cdb16d630fec35d0ea70ce59c17feec16cd1618
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: ebc247f6bf9d895794838ab8b46e0ea71755fe8b159345ab44ceaa17d058892cd2122d589b111a70af0aadcda0d23b072f10d03abcb3fbe61162ef7cb45bb37b
|
7
|
+
data.tar.gz: 134706e21071f54d3badcb97bc04d0a90480f7e10cde16034a90cfac51e7353bb951f98d156e224df1b247b4aed8271c795b7f874f863b906a5cdb3ff3002864
|
data/.gitignore
ADDED
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
MIT License
|
2
|
+
|
3
|
+
Copyright (c) 2020 Emad Elsaid
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
13
|
+
copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
21
|
+
SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,52 @@
|
|
1
|
+
OfflineRSS
|
2
|
+
==========
|
3
|
+
|
4
|
+
Downloads RSS feed URLs and split each entry to an XML file and write them to
|
5
|
+
`~/rss/INBOX`.
|
6
|
+
|
7
|
+
Offlinerss will not write the entry to the inbox directory if it exists in any
|
8
|
+
subdirectory in `~/rss`. This means if you read an article and you want to move
|
9
|
+
it to another directory you can move it to `~/rss/archive` for example and it
|
10
|
+
won't be created again when you run `offlinerss`.
|
11
|
+
|
12
|
+
## Usage
|
13
|
+
|
14
|
+
Clone or copy `offlinerss` to any directory in your `PATH`, put a config file
|
15
|
+
that include your RSS feeds URLs in `~/rss/config.yml` as follows
|
16
|
+
|
17
|
+
```yaml
|
18
|
+
urls:
|
19
|
+
- https://domain.tld/path/to/feed.rss
|
20
|
+
- https://domain.tld/path/to/another/feed.atom
|
21
|
+
```
|
22
|
+
|
23
|
+
Run `offlinerss` when you want to update your RSS feeds.
|
24
|
+
|
25
|
+
## How it works
|
26
|
+
|
27
|
+
- offlinerss will read `~/rss/config.yml`
|
28
|
+
- Will read each URL
|
29
|
+
- For each entry it will write its XML to a file in `~/rss/INBOX/` with a name
|
30
|
+
in this format `sha1(feed url)-sha1(entry id/guid).rss`
|
31
|
+
- If the file `sha1(feed url)-sha1(entry id/guid).rss` exist under any
|
32
|
+
subdirector in `~/rss` it will not be written again.
|
33
|
+
- The rest of the RSS feed is written to `~/rss/.meta` to a file with a name in
|
34
|
+
this format `sha1(feed url).rss`
|
35
|
+
|
36
|
+
|
37
|
+
## Benefits of using the file system as a database
|
38
|
+
|
39
|
+
- Any application can read it
|
40
|
+
- You can search with any file search tool (grep, ag, ripgrep..etc)
|
41
|
+
- You can move the files as you wish to any directory
|
42
|
+
- You can version it with Git or sync it with `rsync`
|
43
|
+
- You can replace `offlinerss` with another implementation and the data doesn't
|
44
|
+
need any transformation, as offlinerss doesn't do any transformation at all,
|
45
|
+
just splits it to file.
|
46
|
+
- You can view the files with any simple application that reads the file and
|
47
|
+
renders it to HTML or any other format
|
48
|
+
|
49
|
+
## Motivation
|
50
|
+
|
51
|
+
I wrote about this script for the first time couple days ago in my blog
|
52
|
+
describing [ why I was set to create it ](https://www.emadelsaid.com/download-RSS-offline/)
|
data/Rakefile
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
require 'bundler/gem_tasks'
|
data/exe/offlinerss
ADDED
@@ -0,0 +1,52 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
# frozen_string_literal: true
|
3
|
+
|
4
|
+
require 'open-uri'
|
5
|
+
require 'fileutils'
|
6
|
+
require 'digest'
|
7
|
+
require 'yaml'
|
8
|
+
require 'rss'
|
9
|
+
|
10
|
+
def mkdir(*paths)
|
11
|
+
path = File.join(*paths)
|
12
|
+
FileUtils.mkdir(path) unless Dir.exist?(path)
|
13
|
+
path
|
14
|
+
end
|
15
|
+
|
16
|
+
destination = mkdir(File.expand_path('~/rss/'))
|
17
|
+
inbox = mkdir(destination, 'INBOX')
|
18
|
+
meta_dir = mkdir(destination, '.meta')
|
19
|
+
|
20
|
+
config_file = File.join(destination, 'config.yml')
|
21
|
+
config = YAML.load_file(config_file)
|
22
|
+
urls = config['urls']
|
23
|
+
|
24
|
+
urls.each do |url|
|
25
|
+
url_digest = Digest::SHA1.hexdigest(url)
|
26
|
+
puts "[Get] #{url}"
|
27
|
+
|
28
|
+
URI.open(url) do |rss|
|
29
|
+
content = rss.read
|
30
|
+
feed = RSS::Parser.parse(content)
|
31
|
+
|
32
|
+
feed.items.each do |item|
|
33
|
+
id = item.respond_to?(:id) ? item.id : item.guid
|
34
|
+
id_digest = Digest::SHA1.hexdigest(id.content)
|
35
|
+
file_basename = url_digest + '-' + id_digest + '.rss'
|
36
|
+
|
37
|
+
next unless Dir.glob(File.join(destination, '**', file_basename)).empty?
|
38
|
+
|
39
|
+
filename = File.join(inbox, file_basename)
|
40
|
+
File.write(filename, item.to_s)
|
41
|
+
end
|
42
|
+
|
43
|
+
[{ start_tag: '<entry>', end_tag: '</entry>' }, { start_tag: '<item>', end_tag: '</item>' }].each do |tag|
|
44
|
+
next unless content.include?(tag[:start_tag])
|
45
|
+
|
46
|
+
content[content.index(tag[:start_tag])...(content.rindex(tag[:end_tag]) + tag[:end_tag].length)] = ''
|
47
|
+
end
|
48
|
+
|
49
|
+
metafile = File.join(meta_dir, url_digest + '.rss')
|
50
|
+
File.write(metafile, content)
|
51
|
+
end
|
52
|
+
end
|
data/offlinerss.gemspec
ADDED
@@ -0,0 +1,27 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
Gem::Specification.new do |spec|
|
4
|
+
spec.name = 'offlinerss'
|
5
|
+
spec.version = '0.1.0'
|
6
|
+
spec.authors = ['Emad Elsaid']
|
7
|
+
spec.email = ['emad.elsaid.hamed@gmail.com']
|
8
|
+
|
9
|
+
spec.summary = 'Download RSS entries to local directory'
|
10
|
+
spec.description = 'Download RSS feed and split them to local directory'
|
11
|
+
spec.homepage = 'https://github.com/emad-elsaid/offlinerss'
|
12
|
+
spec.license = 'MIT'
|
13
|
+
spec.required_ruby_version = Gem::Requirement.new('>= 2.3.0')
|
14
|
+
|
15
|
+
spec.metadata['homepage_uri'] = spec.homepage
|
16
|
+
spec.metadata['source_code_uri'] = 'https://github.com/emad-elsaid/offlinerss'
|
17
|
+
|
18
|
+
# Specify which files should be added to the gem when it is released.
|
19
|
+
# The `git ls-files -z` loads the files in the RubyGem that have been added into git.
|
20
|
+
spec.files = Dir.chdir(File.expand_path(__dir__)) do
|
21
|
+
`git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
|
22
|
+
end
|
23
|
+
spec.bindir = 'exe'
|
24
|
+
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
25
|
+
|
26
|
+
spec.add_runtime_dependency 'rss'
|
27
|
+
end
|
metadata
ADDED
@@ -0,0 +1,68 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: offlinerss
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Emad Elsaid
|
8
|
+
autorequire:
|
9
|
+
bindir: exe
|
10
|
+
cert_chain: []
|
11
|
+
date: 2020-09-06 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: rss
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
27
|
+
description: Download RSS feed and split them to local directory
|
28
|
+
email:
|
29
|
+
- emad.elsaid.hamed@gmail.com
|
30
|
+
executables:
|
31
|
+
- offlinerss
|
32
|
+
extensions: []
|
33
|
+
extra_rdoc_files: []
|
34
|
+
files:
|
35
|
+
- ".gitignore"
|
36
|
+
- Gemfile
|
37
|
+
- Gemfile.lock
|
38
|
+
- LICENSE
|
39
|
+
- README.md
|
40
|
+
- Rakefile
|
41
|
+
- exe/offlinerss
|
42
|
+
- offlinerss.gemspec
|
43
|
+
homepage: https://github.com/emad-elsaid/offlinerss
|
44
|
+
licenses:
|
45
|
+
- MIT
|
46
|
+
metadata:
|
47
|
+
homepage_uri: https://github.com/emad-elsaid/offlinerss
|
48
|
+
source_code_uri: https://github.com/emad-elsaid/offlinerss
|
49
|
+
post_install_message:
|
50
|
+
rdoc_options: []
|
51
|
+
require_paths:
|
52
|
+
- lib
|
53
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
54
|
+
requirements:
|
55
|
+
- - ">="
|
56
|
+
- !ruby/object:Gem::Version
|
57
|
+
version: 2.3.0
|
58
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
59
|
+
requirements:
|
60
|
+
- - ">="
|
61
|
+
- !ruby/object:Gem::Version
|
62
|
+
version: '0'
|
63
|
+
requirements: []
|
64
|
+
rubygems_version: 3.1.2
|
65
|
+
signing_key:
|
66
|
+
specification_version: 4
|
67
|
+
summary: Download RSS entries to local directory
|
68
|
+
test_files: []
|