offlinerss 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: db62105a5c4a87c5f63c8b62a83ec0ca4e9e29de1b912197656c0151204e6545
+  data.tar.gz: 4fc5c9f5af64924987c0dd089cdb16d630fec35d0ea70ce59c17feec16cd1618
+SHA512:
+  metadata.gz: ebc247f6bf9d895794838ab8b46e0ea71755fe8b159345ab44ceaa17d058892cd2122d589b111a70af0aadcda0d23b072f10d03abcb3fbe61162ef7cb45bb37b
+  data.tar.gz: 134706e21071f54d3badcb97bc04d0a90480f7e10cde16034a90cfac51e7353bb951f98d156e224df1b247b4aed8271c795b7f874f863b906a5cdb3ff3002864

data/.gitignore

@@ -0,0 +1,11 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status

data/Gemfile

@@ -0,0 +1,5 @@
+source "https://rubygems.org"
+
+gemspec
+
+gem 'rake'

data/Gemfile.lock

@@ -0,0 +1,21 @@
+PATH
+  remote: .
+  specs:
+    offlinerss (0.1.0)
+      rss
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    rake (13.0.1)
+    rss (0.2.8)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  offlinerss!
+  rake
+
+BUNDLED WITH
+   2.1.4

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 Emad Elsaid
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,52 @@
+OfflineRSS
+==========
+
+Downloads RSS feed URLs and split each entry to an XML file and write them to
+`~/rss/INBOX`.
+
+Offlinerss will not write the entry to the inbox directory if it exists in any
+subdirectory in `~/rss`. This means if you read an article and you want to move
+it to another directory you can move it to `~/rss/archive` for example and it
+won't be created again when you run `offlinerss`.
+
+## Usage
+
+Clone or copy `offlinerss` to any directory in your `PATH`, put a config file
+that include your RSS feeds URLs in `~/rss/config.yml` as follows
+
+```yaml
+urls:
+   - https://domain.tld/path/to/feed.rss
+   - https://domain.tld/path/to/another/feed.atom
+```
+
+Run `offlinerss` when you want to update your RSS feeds.
+
+## How it works
+
+- offlinerss will read `~/rss/config.yml`
+- Will read each URL
+- For each entry it will write its XML to a file in `~/rss/INBOX/` with a name
+  in this format `sha1(feed url)-sha1(entry id/guid).rss`
+- If the file `sha1(feed url)-sha1(entry id/guid).rss` exist under any
+  subdirector in `~/rss` it will not be written again.
+- The rest of the RSS feed is written to `~/rss/.meta` to a file with a name in
+  this format `sha1(feed url).rss`
+
+
+## Benefits of using the file system as a database
+
+- Any application can read it
+- You can search with any file search tool (grep, ag, ripgrep..etc)
+- You can move the files as you wish to any directory
+- You can version it with Git or sync it with `rsync`
+- You can replace `offlinerss` with another implementation and the data doesn't
+  need any transformation, as offlinerss doesn't do any transformation at all,
+  just splits it to file.
+- You can view the files with any simple application that reads the file and
+  renders it to HTML or any other format
+
+## Motivation
+
+I wrote about this script for the first time couple days ago in my blog
+describing [ why I was set to create it ](https://www.emadelsaid.com/download-RSS-offline/)

data/Rakefile

@@ -0,0 +1 @@
+require 'bundler/gem_tasks'

data/exe/offlinerss

@@ -0,0 +1,52 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'open-uri'
+require 'fileutils'
+require 'digest'
+require 'yaml'
+require 'rss'
+
+def mkdir(*paths)
+  path = File.join(*paths)
+  FileUtils.mkdir(path) unless Dir.exist?(path)
+  path
+end
+
+destination = mkdir(File.expand_path('~/rss/'))
+inbox = mkdir(destination, 'INBOX')
+meta_dir = mkdir(destination, '.meta')
+
+config_file = File.join(destination, 'config.yml')
+config = YAML.load_file(config_file)
+urls = config['urls']
+
+urls.each do |url|
+  url_digest = Digest::SHA1.hexdigest(url)
+  puts "[Get] #{url}"
+
+  URI.open(url) do |rss|
+    content = rss.read
+    feed = RSS::Parser.parse(content)
+
+    feed.items.each do |item|
+      id = item.respond_to?(:id) ? item.id : item.guid
+      id_digest = Digest::SHA1.hexdigest(id.content)
+      file_basename = url_digest + '-' + id_digest + '.rss'
+
+      next unless Dir.glob(File.join(destination, '**', file_basename)).empty?
+
+      filename = File.join(inbox, file_basename)
+      File.write(filename, item.to_s)
+    end
+
+    [{ start_tag: '<entry>', end_tag: '</entry>' }, { start_tag: '<item>', end_tag: '</item>' }].each do |tag|
+      next unless content.include?(tag[:start_tag])
+
+      content[content.index(tag[:start_tag])...(content.rindex(tag[:end_tag]) + tag[:end_tag].length)] = ''
+    end
+
+    metafile = File.join(meta_dir, url_digest + '.rss')
+    File.write(metafile, content)
+  end
+end

data/offlinerss.gemspec

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+Gem::Specification.new do |spec|
+  spec.name          = 'offlinerss'
+  spec.version       = '0.1.0'
+  spec.authors       = ['Emad Elsaid']
+  spec.email         = ['emad.elsaid.hamed@gmail.com']
+
+  spec.summary       = 'Download RSS entries to local directory'
+  spec.description   = 'Download RSS feed and split them to local directory'
+  spec.homepage      = 'https://github.com/emad-elsaid/offlinerss'
+  spec.license       = 'MIT'
+  spec.required_ruby_version = Gem::Requirement.new('>= 2.3.0')
+
+  spec.metadata['homepage_uri'] = spec.homepage
+  spec.metadata['source_code_uri'] = 'https://github.com/emad-elsaid/offlinerss'
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+
+  spec.add_runtime_dependency 'rss'
+end

metadata

@@ -0,0 +1,68 @@
+--- !ruby/object:Gem::Specification
+name: offlinerss
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Emad Elsaid
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2020-09-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rss
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Download RSS feed and split them to local directory
+email:
+- emad.elsaid.hamed@gmail.com
+executables:
+- offlinerss
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- Rakefile
+- exe/offlinerss
+- offlinerss.gemspec
+homepage: https://github.com/emad-elsaid/offlinerss
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/emad-elsaid/offlinerss
+  source_code_uri: https://github.com/emad-elsaid/offlinerss
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.3.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.2
+signing_key:
+specification_version: 4
+summary: Download RSS entries to local directory
+test_files: []