enter-rockstar 0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 587022dfb160e25d89d735b881451c94c89b9ca0eeb3228f3e500590f93a9fc2
+  data.tar.gz: 96e1ff6b6eb0e0ce212aa338eeb0ce60c2c460bd69178e822be5d37f45be84c3
+SHA512:
+  metadata.gz: 26395477d080378a0a2d33d3d20307aed8ea23a7f273df94d2c9b538e388337111663a400a20a7c45cf322d93fd802ed3d47d741084575913ebcb97941dfd714
+  data.tar.gz: 0a027225b5e58fab4f20bbf35c8c0f3275bd26ce9427ec1b56507ef562eee9a2c30ce45f472c55b8d615f3979f55c839fdce18d1c564e6e688fe87dd4c823e64

data/.gitignore

@@ -0,0 +1,13 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status
+.DS_Store
+*.gem

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,11 @@
+AllCops:
+  TargetRubyVersion: 2.3
+
+Metrics:
+  Enabled: false
+
+Style/RescueModifier:
+  Enabled: false
+
+Lint/InterpolationCheck:
+  Enabled: false

data/.ruby-version

@@ -0,0 +1 @@
+2.6.2

data/.travis.yml

@@ -0,0 +1,7 @@
+---
+sudo: false
+language: ruby
+cache: bundler
+rvm:
+  - 2.6.2
+before_install: gem install bundler -v 2.0.1

data/Gemfile

@@ -0,0 +1,4 @@
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in enter-rockstar.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,79 @@
+PATH
+  remote: .
+  specs:
+    enter-rockstar (0.1)
+      nokogiri
+      thor
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.6.0)
+      public_suffix (>= 2.0.2, < 4.0)
+    ast (2.4.0)
+    coderay (1.1.2)
+    crack (0.4.3)
+      safe_yaml (~> 1.0.0)
+    diff-lcs (1.3)
+    hashdiff (0.3.8)
+    jaro_winkler (1.5.2)
+    method_source (0.9.2)
+    mini_portile2 (2.4.0)
+    nokogiri (1.10.2)
+      mini_portile2 (~> 2.4.0)
+    parallel (1.15.0)
+    parser (2.6.2.0)
+      ast (~> 2.4.0)
+    pry (0.12.2)
+      coderay (~> 1.1.0)
+      method_source (~> 0.9.0)
+    psych (3.1.0)
+    public_suffix (3.0.3)
+    rainbow (3.0.0)
+    rake (12.3.2)
+    rspec (3.8.0)
+      rspec-core (~> 3.8.0)
+      rspec-expectations (~> 3.8.0)
+      rspec-mocks (~> 3.8.0)
+    rspec-core (3.8.0)
+      rspec-support (~> 3.8.0)
+    rspec-expectations (3.8.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-mocks (3.8.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-support (3.8.0)
+    rubocop (0.66.0)
+      jaro_winkler (~> 1.5.1)
+      parallel (~> 1.10)
+      parser (>= 2.5, != 2.5.1.1)
+      psych (>= 3.1.0)
+      rainbow (>= 2.2.2, < 4.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 1.6)
+    ruby-progressbar (1.10.0)
+    safe_yaml (1.0.5)
+    thor (0.20.3)
+    unicode-display_width (1.5.0)
+    vcr (4.0.0)
+    webmock (3.5.1)
+      addressable (>= 2.3.6)
+      crack (>= 0.3.2)
+      hashdiff
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 2.0)
+  enter-rockstar!
+  pry
+  rake
+  rspec
+  rubocop
+  vcr
+  webmock
+
+BUNDLED WITH
+   2.0.1

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2019 Marcin Ruszkiewicz
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,64 @@
+[![Build Status](https://travis-ci.org/marcinruszkiewicz/enter-rockstar.svg?branch=master)](https://travis-ci.org/marcinruszkiewicz/enter-rockstar)
+[![Gem Version](https://badge.fury.io/rb/enter-rockstar.svg)](https://badge.fury.io/rb/enter-rockstar)
+
+# Enter Rockstar - a tool to help with programming in Rockstar
+
+This is a set of tools that help Rockstar programmers create programs in the [Rockstar language](https://github.com/RockstarLang/rockstar).
+
+For details on what is done and what I'm still working on, see the TODO.md and CHANGELOG.md files.
+
+## Installation
+
+Install the gem by issuing the following command.
+
+```
+$ gem install enter-rockstar
+```
+
+This gem works best on a current Ruby version and requires Ruby 2.3 at minimum. Running it on 2.3 has the downside of metal umlauts not being entirely correct as that Ruby version doesn't know how to `.downcase` a capital umlaut letter, which was fixed in 2.4.
+
+If you're not using the umlauts (or at least are careful to only replace lowercase letters with them), all should be fine otherwise.
+
+## Setup
+
+To start working with Enter Rockstar, you will need a word base to generate new lyrics from. To help with that task, the gem includes word data created from the "Heavy Metal" category, which should cover most common lyrics, however you can import a different category if you want.
+
+### Scraping a Wikia category page
+
+You have to start with creating a list of all the pages in the Wikia's category, for example like this:
+
+```
+$ enter-rockstar scrape_category power_metal /wiki/Category:Genre/Power_Metal
+```
+
+This will create a file in `lyrics_data/wikia_power_metal.json` with all the links to pages in this category. Next you want to scrape actual lyric pages based on the links in the json file:
+
+```
+$ enter-rockstar scrape_lyrics power_metal
+```
+
+After this command finishes (which might take a long time depending on what category you use), you will have a set of directories with text files in the `lyrics` directory.
+
+### Generating a word base
+
+Now that you have a set of lyrics, it's time to convert them into something that Enter Rockstar can use.
+
+## Generating lyrics
+
+### Finding words for poetic literals
+
+The most common and basic function of Enter Rockstar is just finding interesting words of good length to use in the poetic numeral representations. Finding out what words to use to represent `123` is not as easy as it might sound and this makes it easier:
+
+```
+$ enter-rockstar poetic 123
+```
+
+
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/marcinruszkiewicz/enter-rockstar. I'm also available for questions at the [Rockstar Developers Discord Group](https://discord.gg/kEUe5bM)
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec

data/bin/console

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'enter_rockstar'
+require 'pry'
+
+EnterRockstar.scrape_lyrics

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/enter-rockstar.gemspec

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+lib = File.expand_path('lib', __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'enter_rockstar/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'enter-rockstar'
+  spec.version       = EnterRockstar::VERSION
+  spec.authors       = ['Marcin Ruszkiewicz']
+  spec.email         = ['marcin.ruszkiewicz@polcode.net']
+
+  spec.summary       = 'Generating helpful Rock phrases to make programming in Rockstar easier.'
+  spec.homepage      = 'https://github.com/marcinruszkiewicz/enter-rockstar'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(spec|examples)/})
+  end
+  spec.bindir        = 'exe'
+  spec.executables   = ['enter-rockstar']
+  spec.require_paths = ['lib']
+
+  spec.required_ruby_version = '>= 2.3'
+
+  spec.add_development_dependency 'bundler', '~> 2.0'
+  spec.add_development_dependency 'pry'
+  spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'rspec'
+  spec.add_development_dependency 'rubocop'
+  spec.add_development_dependency 'vcr'
+  spec.add_development_dependency 'webmock'
+  # spec.add_dependency 'hashie'
+  spec.add_dependency 'thor'
+  spec.add_dependency 'nokogiri'
+end

data/exe/enter-rockstar

@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'rubygems'
+require 'bundler/setup'
+require 'enter_rockstar'
+require 'enter_rockstar/cli'
+
+EnterRockstar::CLI.start(ARGV)

data/lib/enter_rockstar/cli.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require 'thor'
+require_relative 'version'
+
+module EnterRockstar
+  # command line interface for enter-rockstar command
+  class CLI < Thor
+    package_name "Enter-Rockstar v#{EnterRockstar::VERSION}"
+
+    desc 'scrape_category CATEGORY_NAME URL', 'scrape lyrics wikia category page for bands and albums'
+    def scrape_category(category_name, url)
+      scraper = EnterRockstar::Scraper::Wikia.new(category_name: category_name, url: url)
+      scraper.parse_category(test_limit: true)
+      scraper.save_category
+      say
+    end
+
+    desc 'scrape_lyrics CATEGORY_NAME START_INDEX', 'scrape actual lyrics from the lyrics wikia using the generated json file'
+    def scrape_lyrics(category_name, start_index=0)
+      scraper = EnterRockstar::Scraper::Wikia.new(category_name: category_name)
+      scraper.load_saved_json
+      scraper.parse_all_pages(start_index: start_index)
+    end
+
+    desc 'print_indexed_tree CATEGORY_NAME', 'print tree with indexes'
+    def print_indexed_tree(category_name)
+      scraper = EnterRockstar::Scraper::Wikia.new(category_name: category_name)
+      scraper.load_saved_json
+      scraper.print_indexed_tree
+    end
+
+    desc 'tokenize DATA_DIR', 'take the downloaded lyrics text files and tokenize them'
+    def tokenize(data_dir)
+      tokenizer = EnterRockstar::Corpus::Tokenizer.new(data_dir: data_dir)
+      tokenizer.tokenize
+    end
+
+    desc 'poetic NUMBER', 'generate a poetic representation of a number from the word base'
+    def poetic(number)
+      generator = EnterRockstar::Generator::Poetic.new
+      results = generator.number(123)
+
+      say results
+    end
+  end
+end

data/lib/enter_rockstar/corpus/tokenizer.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module EnterRockstar
+  module Corpus
+    # take the downloaded lyrics texts and tokenize them
+    class Tokenizer
+      def initialize(data_dir: )
+        @data_dir = data_dir
+      end
+
+      def tokenize
+
+      end
+    end
+  end
+end

data/lib/enter_rockstar/generator/poetic.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module EnterRockstar
+  module Generator
+    # lyrics scraper for lyrics.wikia.com
+    class Poetic
+      def initialize
+
+      end
+
+      def number(number)
+
+      end
+    end
+  end
+end

data/lib/enter_rockstar/scraper/wikia.rb

@@ -0,0 +1,147 @@
+# frozen_string_literal: true
+
+require 'open-uri'
+require 'nokogiri'
+require 'json'
+
+module EnterRockstar
+  module Scraper
+    # lyrics scraper for lyrics.wikia.com
+    class Wikia
+      START_HOST = 'http://lyrics.wikia.com'
+      DATA_DIR = 'lyrics'
+      SLEEP_BETWEEN_REQUESTS = 0.1
+
+      attr_reader :tree
+
+      def initialize(category_name: 'heavy_metal', url: '/wiki/Category:Genre/Heavy_Metal', data_dir: 'lyrics_data')
+        @tree = {}
+        @output = "#{data_dir}/wikia_#{category_name}.json"
+        @url = url
+        @category_name = category_name
+      end
+
+      def parse_category(url: nil, test_limit: false)
+        url ||= START_HOST + @url
+        html = URI.open(url)
+        doc = Nokogiri::HTML(html)
+
+        # get all category member links and sort them by band and album
+        doc.css('li.category-page__member a').each do |category_link|
+          next if category_link.attr('title').include?('Category:')
+
+          band, album = category_link.attr('title').split(':')
+          @tree[band] ||= {}
+
+          if album.nil?
+            @tree[band]['band_url'] = category_link.attr('href')
+          else
+            @tree[band][album] = category_link.attr('href')
+          end
+        end
+
+        return if test_limit # test only first page scraping so it's easier
+
+        print '.'
+        # get next page if one exists and parse that
+        next_url = doc.css('a.category-page__pagination-next')&.first&.attr('href')
+        parse_category(url: next_url) unless next_url.nil?
+      end
+
+      def save_category
+        puts
+        out = File.new(@output, 'w')
+        out.write @tree.to_json
+        out.close
+        puts "Saved JSON data to #{@output}"
+      end
+
+      def load_saved_json
+        file = File.read(@output)
+        @tree = JSON.parse(file)
+        @new_tree = JSON.parse(file)
+      end
+
+      def print_indexed_tree
+        @tree.each_with_index do |(key, _val), index|
+          puts "#{index}: #{key}"
+        end
+      end
+
+      def parse_all_pages(start_index: 0)
+        @tree.each_with_index do |(key, val), index|
+          next if index < start_index
+
+          puts "#{index}: #{key}"
+
+          val.each do |k, v|
+            dirname = k == 'band_url' ? [DATA_DIR, @category_name, key].join('/') : [DATA_DIR, @category_name, key, k].join('/')
+            FileUtils.mkdir_p dirname
+
+            parse_page(v, dirname, key)
+          end
+        end
+
+        @tree = @new_tree
+        save_category
+      end
+
+      def parse_page(url, dirname, band)
+        puts url
+        sleep SLEEP_BETWEEN_REQUESTS
+        html = URI.open(START_HOST + url)
+        doc = Nokogiri::HTML(html)
+
+        if doc.css('h2 span.mw-headline a').count.zero?
+          # single album page listed on the category
+          doc.css('div.mw-content-text ol li a').each do |song|
+            parse_song(song.attr('href'), dirname, song.text) if song&.attr('href')
+          end
+          puts
+        else
+          doc.css('h2 span.mw-headline a').each do |album|
+            puts album.text
+            # some band pages have extra albums that are not listed in the category page for some reason
+            album_dirname = [dirname, album.text].join('/')
+            FileUtils.mkdir_p album_dirname
+            @new_tree[band][album.text] = album.attr('href')
+
+            # get song pages
+            album.parent.parent.css('+ div + ol > li a').each do |song|
+              parse_song(song.attr('href'), album_dirname, song.text) if song&.attr('href')
+            end
+            puts
+          end
+        end
+      end
+
+      def parse_song(url, dirname, songname)
+        return if url.start_with? 'http'
+
+        songfile = "#{dirname}/#{songname}.txt"
+        without_last = songfile.split('/')
+        without_last.pop
+        FileUtils.mkdir_p without_last.join('/')
+        return if File.exist?(songfile)
+
+        print '.'
+        sleep SLEEP_BETWEEN_REQUESTS
+        html = URI.open(START_HOST + url)
+        doc = Nokogiri::HTML(html)
+
+        lyrics = doc.css('div.lyricbox').first
+        return if lyrics.nil?
+
+        if lyrics.css('a')&.first&.attr('href') == '/wiki/Category:Instrumental'
+          # instrumental song, whatever
+        else
+          proper_text = lyrics.inner_html.gsub(%r{<div.*?(\/div>)}, '').split('<br>').join("\n")
+
+          out = File.new(songfile, 'w')
+          out.write proper_text
+          out.close
+        end
+      end
+    end
+  end
+end

data/lib/enter_rockstar/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module EnterRockstar
+  VERSION = '0.1'
+end

data/lib/enter_rockstar.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require 'enter_rockstar/scraper/wikia'
+require 'pry'
+
+module EnterRockstar
+
+end