ocremix_parser 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 7877e9a0233d31be28de858799f82b05b89dd112
+  data.tar.gz: 9c1d128a96c48ad365834e9956596889623725f5
+SHA512:
+  metadata.gz: 74ababfbc361fb7bf30191518961a8eeabad0c3d0932d7cf634b0ed0b3344bd31f269c6d2810962fa4880db22e2bf4757ded2047560b11ba9bcb7dba931ca01d
+  data.tar.gz: 788a06eb92b852b92ba34cfbbd1568887b77f885b67e46b85834961136a3c5e6205cabf1b812b25dded0f51ef97e411b32e5cdcc7b584ee6dd1f333a28a9e9cb

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.3.1
+before_install: gem install bundler -v 1.12.4

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in ocremix_parser.gemspec
+gemspec

data/README.md

@@ -0,0 +1,17 @@
+# OcremixParser
+
+This gem will download the top 10 tracks from ocremix.org to where ever you configure it to do so via `config/application.yml`
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+
+Or install it yourself as:
+
+    $ gem install ocremix_parser
+
+## Usage
+
+For right now... don't actually install the gem, just use ./bin/console, which will begin the downloading process
+

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "ocremix_parser"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+puts "ONLY USING TMP/ BUILT IN CONFIG FILE"
+ENV['HOME'] = "/tmp/ocremix_parser_test_home"
+
+mg = OcremixParser::MixGrabber.new
+mg.download_ten_latest_mixes_via_web_scrapes
+

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/config/application.yml

@@ -0,0 +1,4 @@
+rss_feed: "http://ocremix.org/feeds/ten20/"
+file_mirrors: "http://ocr.blueblue.fr/files/music/remixes http://iterations.org/files/music/remixes http://ocrmirror.org/files/music/remixes"
+
+download_directory: "/storage/storage/ocremix"

data/exe/ocremix_parser

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+
+require 'ocremix_parser'
+
+# TODO: add thor and use interface like:  list, start 
+
+mg = OcremixParser::MixGrabber.new
+
+mg.download_ten_latest_mixes_via_web_scrapes
+

data/lib/ocremix_parser.rb

@@ -0,0 +1,118 @@
+require 'i18n'
+require 'figaro'
+require 'open-uri'
+require 'simple-rss'
+require 'nokogiri'
+require 'fileutils'
+
+require "ocremix_parser/version"
+
+module OcremixParser
+
+  class MixGrabber
+
+    def initialize
+      prepare_config_file!
+
+      @user_agent = '"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:47.0) Gecko/20100101 Firefox/47.0"'
+      @mirror = ENV['file_mirrors'].split(" ").last
+
+      @skips = 0
+      @downloads = 0
+    end
+
+    def prepare_config_file!
+      bundled_config_path = File.expand_path('../../config/application.yml', __FILE__)
+      config_path = "#{ENV['HOME']}/.config/ocremix_parser.yml"
+
+      # create the config file unless it exists
+      unless File.exists?(config_path)
+        FileUtils.mkdir_p File.dirname(config_path)
+        FileUtils.cp bundled_config_path, config_path
+      end
+
+      Figaro.application.path = config_path
+      Figaro.load
+    end
+
+    def download_ten_latest_mixes_via_web_scrapes
+      top_ten_track_page_links = query_from_top_ten_rss_feed(:link)
+
+      top_ten_track_page_links.each do |page_url|
+        dl_links = convert_track_page_to_mp3_links(page_url)
+        file_name = File.basename(dl_links.first)
+        destination_path = "#{ENV['download_directory']}/#{file_name}"
+
+        wget_download(dl_links.first, destination_path)
+      end
+
+      puts "#{@downloads} downloads happened, #{@skips} skips"
+    end
+
+    def download_ten_latest_remixes_via_filename_guessing
+      # pull down the latest file_names from the rss feed
+      top_ten_file_names = query_from_top_ten_rss_feed.map {|s| convert_title_to_filename(s)}
+
+      top_ten_file_names.each do |file_name|
+        url = "#{@mirror}/#{file_name}"
+        destination_path = "#{ENV['download_directory']}/#{file_name}"
+
+        wget_download(url, destination_path)
+      end
+    end
+
+    def wget_download(url, destination_path)
+      if File.exists? destination_path
+        @skips += 1
+        return
+      end
+
+      @downloads += 1
+
+      # Download with a progress bar...
+      # Wget user agent required (I have an odd firewall...)!
+      cmd = "wget #{url} -O #{destination_path} -U #{@user_agent}"
+      puts cmd
+      `#{cmd}`
+    end
+
+    def convert_title_to_filename(string)
+      tag = "_OC_ReMix.mp3"
+      string = convert_foriegn_characters_to_en(string)
+
+      string.gsub(" ", "_").gsub(/[^0-9A-Za-z_\-]/, "") + tag
+    end
+
+    def convert_foriegn_characters_to_en(string)
+      I18n.available_locales = [:en]
+      string = I18n.transliterate(string.force_encoding('utf-8'))
+    end
+
+    def query_from_top_ten_rss_feed(key_to_query = :title)
+      rss = SimpleRSS.parse open(ENV['rss_feed'])
+      rss.entries.collect {|e| e[key_to_query] }
+    end
+
+    # Not Used... handy if wget doesn't exist though...
+    def download_bad_way(destination_path, source_path, user_agent)
+
+      # Download without a progress bar
+      File.open(destination_path, "wb") do |saved_file|
+        # the following "open" is provided by open-uri
+        open("#{source_path}", "rb", 'User-Agent' => useragent) do |read_file|
+          saved_file.write(read_file.read)
+        end
+      end
+    end
+
+    # e.g. pass in http://ocremix.org/remix/OCR03341
+    # returns [ "http://mirrorname.com/file/path/File_Name.mp3", ... ]
+    def convert_track_page_to_mp3_links(url)
+      doc = Nokogiri.parse( open(url, "rb", 'User-Agent' => @user_agent).read )
+      li_mirrors = doc.css("#panel-download > div:nth-child(1) > ul:nth-child(4) > li")
+      download_links = li_mirrors.collect {|li| li.css("a").first["href"] }
+    end
+
+  end
+
+end

data/lib/ocremix_parser/version.rb

@@ -0,0 +1,3 @@
+module OcremixParser
+  VERSION = "0.1.0"
+end

data/ocremix_parser.gemspec

@@ -0,0 +1,29 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'ocremix_parser/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "ocremix_parser"
+  spec.version       = OcremixParser::VERSION
+  spec.authors       = ["TheNotary"]
+  spec.email         = ["no@email.plz"]
+
+  spec.summary       = %q{This .}
+  spec.description   = %q{: Write a longer description or delete this line.}
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "figaro"
+  spec.add_dependency "simple-rss"
+  spec.add_dependency "i18n"
+  spec.add_dependency "nokogiri"
+
+  spec.add_development_dependency "bundler", "~> 1.12"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "pry"
+end

metadata

@@ -0,0 +1,169 @@
+--- !ruby/object:Gem::Specification
+name: ocremix_parser
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- TheNotary
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2016-07-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: figaro
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: simple-rss
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: i18n
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.12'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.12'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: ": Write a longer description or delete this line."
+email:
+- no@email.plz
+executables:
+- ocremix_parser
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- config/application.yml
+- exe/ocremix_parser
+- lib/ocremix_parser.rb
+- lib/ocremix_parser/version.rb
+- ocremix_parser.gemspec
+homepage: 
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: This .
+test_files: []