feed_into 0.2.9

data/lib/modules/general.rb

@@ -0,0 +1,250 @@
+module General
+    def crl_general( sym, cmd, channel, response, data, obj, custom={} )
+      messages = []
+  
+      case sym
+        when :download
+          result, messages = crl_general_download( cmd[:url], obj )
+        when :pre_titles
+          result, messages = crl_general_pre_titles( cmd, channel, data, obj )
+        when :mining_rss_one
+          result = crl_general_mining_rss_one( cmd[:url], response, obj )
+        when :mining_rss_two
+          result = crl_general_mining_rss_two( cmd[:url], response, obj )
+        when :format_url_s3
+          result = crl_general_format_url_s3( obj, channel[:options][:html], custom[:query] )
+        when :format_html_remove
+          result = crl_general_format_html_remove( custom[:html] )
+      else
+        messages.push( "General: #{sym} not found." )
+      end
+  
+      return result, messages
+    end
+  
+  
+    private
+  
+  
+    def crl_general_channels()
+      return []
+    end
+  
+  
+    def crl_general_download( url, obj )
+      version = ( rand( 89.0..91.0 ) + ( rand( 530.0..540.0 ) / 1000 ) ).round( 2 ) 
+      agent = obj[:format][:download][:agent].gsub( '{{version}}', version.to_s )
+      uri = URI( url )
+
+      header = {}
+      header['User-Agent'] = agent
+      header['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
+      header['Accept-Language'] = 'en-US,en;q=0.5'
+      header['Connection'] = 'keep-alive'
+      header['Upgrade-Insecure-Requests'] = '1'
+      header['Sec-Fetch-Dest'] = 'document'
+      header['Sec-Fetch-Mode'] = 'navigate'
+      header['Sec-Fetch-Site'] = 'none'
+      header['Sec-Fetch-User'] = '?1'
+      header['Pragma'] = 'no-cache'
+      header['Cache-Control'] = 'no-cache'
+
+      response = Net::HTTP.get_response( uri, header )
+      return response.body, [ "Download: Status #{response.code}" ]
+    end
+    
+    
+    def crl_general_mining_rss_one( url, response, obj )
+      doc = Nokogiri::XML( response )
+  
+      feed = {
+          meta: {
+            title: nil,
+            url: nil
+          },
+          items: []
+      }
+  
+      feed[:meta][:title] = doc.at( 'title' ).text.gsub( '"',"'" )
+      feed[:meta][:url] = url
+  
+      entries = doc.css( 'item' )
+      entries.each do | entry | 
+        item = {
+          title: nil,
+          time: {
+            stamp: nil,
+            utc: nil
+          }
+        }
+  
+        tmp = entry.at( 'title' ).text
+        item[:title] = self
+          .method( 'crl_general' )
+          .call( :format_html_remove, nil, nil, nil, nil, nil, { html: tmp } )[ 0 ]
+  
+        item[:title_viewer] = item[:title]
+        item[:time][:stamp] = Time.parse( entry.at( 'pubDate' ) ).to_i
+        item[:time][:utc] = entry.at( 'pubDate' ).text
+        item[:url] = entry.at( 'link' ).text
+  
+        feed[:items].push( item )
+      end
+  
+      return feed
+    end
+    
+    
+    def crl_general_mining_rss_two( url, response, obj )
+      doc = Nokogiri::XML( response )
+  
+      feed = {
+          meta: {
+            title: nil,
+            url: nil
+          },
+          items: []
+      }
+  
+      feed[:meta][:title] = doc.at( 'title' ).text.gsub( '"',"'" )
+      feed[:meta][:url] = url
+  
+      entries = doc.css( 'entry' )
+      entries.each do | entry | 
+        item = {
+          title: nil,
+          time: {
+            stamp: nil,
+            utc: nil
+          }
+        }
+  
+        tmp = entry.at( 'title' ).text
+        item[:title] = self
+          .method( 'crl_general' )
+          .call( :format_html_remove, nil, nil, nil, nil, nil, { html: tmp } )[ 0 ]
+  
+        item[:title_viewer] = item[:title]
+        item[:time][:stamp] = Time.parse( entry.at( 'updated' ) ).to_i
+        item[:time][:utc] = entry.at( 'updated' ).text
+        item[:url] = entry.at( 'link' ).attribute('href').value
+  
+        feed[:items].push( item )
+      end
+  
+      return feed
+    end
+    
+    
+    def crl_general_format_url_s3( obj, file, query )
+      result = ''
+      result << 'https://'
+      result << obj[:options][:s3][:bucket_name]
+      result << '.s3.'
+      result << obj[:options][:s3][:region]
+      result << '.amazonaws.com/'
+      result << obj[:options][:s3][:bucket_sub_folder]
+      result << obj[:options][:s3][:bucket_folder]
+      result << file
+      result << '?'
+      result << URI.encode_www_form( query )
+  
+      return result
+    end
+    
+    
+    def crl_general_format_html_remove( html )
+  
+      result = ''
+      Nokogiri::HTML( CGI.unescapeHTML( html.to_s ) ).traverse do | e |
+        result << e.text if e.text?
+      end
+  
+      result = result
+        .strip
+        .split( ' ' )
+        .map{ | word | word.capitalize }
+        .join( ' ' )
+  
+      return result
+    end
+    
+  
+    def crl_general_pre_titles( cmd, channel, data, obj )
+      messages = []
+  
+      data[:items].map.with_index do | item, index |
+        title, errors = crl_general_pre_title( cmd, channel, data, index, obj )
+        messages.concat( errors )
+        item[:title] = title
+      end
+      
+      return data, messages
+    end
+    
+    
+    def crl_general_pre_title( cmd, channel, data, d_index, obj )
+      messages = []
+      str = obj[:format][:title][:str]
+  
+      parts = str
+        .scan( /\{{[a-z,_,:]+\}}/ )
+        .map { | match | 
+          {
+            gsub: match,
+            cmd: match.gsub( /[{:}]/, '' )
+          }
+        }
+  
+      parts.each do | part |
+        text = part[:cmd].to_sym
+        formats = []
+        
+        if !part[:cmd].index( '__' ).nil?
+          tmp = part[:cmd].split( '__' )
+          formats = tmp.last.split( '_' ).map { | a | a.to_sym }
+          text = tmp[ 0 ].to_sym
+        end
+  
+        case text
+          when :cmd_name
+            insert = cmd[:name].dup.to_s
+          when :channel_name
+            insert = channel[:name].dup.to_s.gsub( '_', ' ' )
+          when :sym
+            insert = obj[:format][:title][:symbol][ channel[:sym] ].dup
+          when :separator
+            insert = obj[:format][:title][:separator].dup
+          when :title_channel
+            insert = channel[:name].dup
+          when :title_item
+            insert = data[:items][ d_index ][:title].dup
+          when :title_meta
+            insert = data[:meta][:title].dup
+        else
+          messages.push( "Set Title (insert): #{text} not found." )
+        end
+  
+        formats.each do | f |
+          case f
+            when :upcase
+              f.eql?( :upcase ) ? insert.upcase! : ''
+            when :titleize
+              insert = insert
+                .split( ' ' )
+                .map { | word | word.capitalize }
+                .join( ' ' )
+          else
+            messages.push( "Set Title (format): #{text} not found." )
+          end
+        end
+        str = str.gsub( part[:gsub], insert )
+      end
+  
+      if str.length > obj[:format][:title][:length]
+        str = str[ 0, obj[:format][:title][:length] ] + obj[:format][:title][:more]
+      end
+  
+      return str, messages
+    end
+  end

metadata

@@ -0,0 +1,176 @@
+--- !ruby/object:Gem::Specification
+name: feed_into
+version: !ruby/object:Gem::Version
+  version: 0.2.9
+platform: ruby
+authors:
+- a6b8
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2023-04-13 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.14.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.14.2
+- !ruby/object:Gem::Dependency
+  name: time
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+- !ruby/object:Gem::Dependency
+  name: tzinfo
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.0.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.0.4
+- !ruby/object:Gem::Dependency
+  name: cgi
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.0
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.5.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.5.1
+- !ruby/object:Gem::Dependency
+  name: uri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.12.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.12.1
+- !ruby/object:Gem::Dependency
+  name: net-http
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.1
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.1'
+description: Merge multiple different data streams into a custom structure. Also easy
+  to expand by a custom module system.
+email:
+- hello@13plus4.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".circleci/config.yml"
+- ".gitignore"
+- ".rubocop.yml"
+- CHANGELOG.md
+- CODE_OF_CONDUCT.md
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- feed_into.gemspec
+- lib/feed_into.rb
+- lib/feed_into/version.rb
+- lib/modules/general.rb
+homepage: https://github.com/a6b8/feed-into-for-ruby
+licenses:
+- MIT
+metadata:
+  allowed_push_host: https://rubygems.org
+  homepage_uri: https://github.com/a6b8/feed-into-for-ruby
+  source_code_uri: https://github.com/a6b8/feed-into-for-ruby
+  changelog_uri: https://raw.githubusercontent.com/a6b8/feed-into-for-ruby/main/CHANGELOG.md
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.4.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.2.3
+signing_key:
+specification_version: 4
+summary: Merge multiple different data streams into a custom structure.
+test_files: []