enter-rockstar 0.1 → 0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 587022dfb160e25d89d735b881451c94c89b9ca0eeb3228f3e500590f93a9fc2
-  data.tar.gz: 96e1ff6b6eb0e0ce212aa338eeb0ce60c2c460bd69178e822be5d37f45be84c3
+  metadata.gz: ea03c06ad39dbfe4e9da6108fef07a4d7f45a6b81900d20d24067eb634f2c5b9
+  data.tar.gz: 22276980116c4dfd719162ebab301eb44fd4616739a79c26aac061da13c92aa4
 SHA512:
-  metadata.gz: 26395477d080378a0a2d33d3d20307aed8ea23a7f273df94d2c9b538e388337111663a400a20a7c45cf322d93fd802ed3d47d741084575913ebcb97941dfd714
-  data.tar.gz: 0a027225b5e58fab4f20bbf35c8c0f3275bd26ce9427ec1b56507ef562eee9a2c30ce45f472c55b8d615f3979f55c839fdce18d1c564e6e688fe87dd4c823e64
+  metadata.gz: d7f7a0fc4ca05c53f6c102f868fed24f74fa466bd6d96bea8f15dcd38b36db3bfb445557f3e79c440717738457f7d3fb069245e50c382bc09f4e41216a60191f
+  data.tar.gz: 141e7a57e51358471d5c4b662833e47c2109cf98f3c625698afe26b7695a039bbd37744e6ea745059b8bb39645f7fccaaa422abeaf648a26b393d5660e3df65d

data/.gitignore

@@ -11,3 +11,11 @@
 .rspec_status
 .DS_Store
 *.gem
+
+# lyrics data
+lyrics/
+lyrics.zip
+
+# rspec temp for downloaded files
+spec/tmp/**
+!spec/tmp/.keep

data/.travis.yml

@@ -1,7 +1,10 @@
----
 sudo: false
 language: ruby
-cache: bundler
 rvm:
+  - 2.3.8
+  - 2.4.5
+  - 2.5.5
   - 2.6.2
-before_install: gem install bundler -v 2.0.1
+before_install:
+  - gem install bundler
+  - gem update --system

data/CHANGELOG.md

@@ -0,0 +1,7 @@
+# 0.2
+
+- [x] security update of nokogiri
+
+# 0.1
+
+- [x] Initial kinda working gem to camp on the rubygems name.

data/Gemfile.lock

@@ -3,64 +3,67 @@ PATH
   specs:
     enter-rockstar (0.1)
       nokogiri
+      progressbar
       thor
+      whatlanguage
 
 GEM
   remote: https://rubygems.org/
   specs:
-    addressable (2.6.0)
-      public_suffix (>= 2.0.2, < 4.0)
+    addressable (2.7.0)
+      public_suffix (>= 2.0.2, < 5.0)
     ast (2.4.0)
     coderay (1.1.2)
     crack (0.4.3)
       safe_yaml (~> 1.0.0)
     diff-lcs (1.3)
-    hashdiff (0.3.8)
-    jaro_winkler (1.5.2)
+    fakefs (0.20.1)
+    hashdiff (1.0.0)
+    jaro_winkler (1.5.3)
     method_source (0.9.2)
     mini_portile2 (2.4.0)
-    nokogiri (1.10.2)
+    nokogiri (1.10.4)
       mini_portile2 (~> 2.4.0)
-    parallel (1.15.0)
-    parser (2.6.2.0)
+    parallel (1.17.0)
+    parser (2.6.4.1)
       ast (~> 2.4.0)
+    progressbar (1.10.1)
     pry (0.12.2)
       coderay (~> 1.1.0)
       method_source (~> 0.9.0)
-    psych (3.1.0)
-    public_suffix (3.0.3)
+    public_suffix (4.0.1)
     rainbow (3.0.0)
-    rake (12.3.2)
+    rake (12.3.3)
     rspec (3.8.0)
       rspec-core (~> 3.8.0)
       rspec-expectations (~> 3.8.0)
       rspec-mocks (~> 3.8.0)
-    rspec-core (3.8.0)
+    rspec-core (3.8.2)
       rspec-support (~> 3.8.0)
-    rspec-expectations (3.8.2)
+    rspec-expectations (3.8.4)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.8.0)
-    rspec-mocks (3.8.0)
+    rspec-mocks (3.8.1)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.8.0)
-    rspec-support (3.8.0)
-    rubocop (0.66.0)
+    rspec-support (3.8.2)
+    rubocop (0.74.0)
       jaro_winkler (~> 1.5.1)
       parallel (~> 1.10)
-      parser (>= 2.5, != 2.5.1.1)
-      psych (>= 3.1.0)
+      parser (>= 2.6)
       rainbow (>= 2.2.2, < 4.0)
       ruby-progressbar (~> 1.7)
-      unicode-display_width (>= 1.4.0, < 1.6)
-    ruby-progressbar (1.10.0)
+      unicode-display_width (>= 1.4.0, < 1.7)
+    ruby-progressbar (1.10.1)
     safe_yaml (1.0.5)
     thor (0.20.3)
-    unicode-display_width (1.5.0)
-    vcr (4.0.0)
-    webmock (3.5.1)
+    unicode-display_width (1.6.0)
+    vcr (5.0.0)
+    webmock (3.7.5)
       addressable (>= 2.3.6)
       crack (>= 0.3.2)
-      hashdiff
+      hashdiff (>= 0.4.0, < 2.0.0)
+    whatlanguage (1.0.6)
 
 PLATFORMS
   ruby
@@ -68,6 +71,7 @@ PLATFORMS
 DEPENDENCIES
   bundler (~> 2.0)
   enter-rockstar!
+  fakefs
   pry
   rake
   rspec

data/README.md

@@ -1,10 +1,24 @@
-[![Build Status](https://travis-ci.org/marcinruszkiewicz/enter-rockstar.svg?branch=master)](https://travis-ci.org/marcinruszkiewicz/enter-rockstar)
+[![Build Status](https://travis-ci.com/marcinruszkiewicz/enter-rockstar.svg?branch=master)](https://travis-ci.com/marcinruszkiewicz/enter-rockstar)
 [![Gem Version](https://badge.fury.io/rb/enter-rockstar.svg)](https://badge.fury.io/rb/enter-rockstar)
 
 # Enter Rockstar - a tool to help with programming in Rockstar
 
 This is a set of tools that help Rockstar programmers create programs in the [Rockstar language](https://github.com/RockstarLang/rockstar).
 
+Basically this allows you (with some setup) to do something like this:
+
+```
+$ enter-rockstar poetic 561 lyrics_data/sentenced_tokens.json
+agony faster desperation
+swept dismay mothafuckin
+knees desire melancholic
+fears facial destruction
+doing sweats generations
+
+```
+
+Which helps greatly with writing cool looking Rockstar programs.
+
 For details on what is done and what I'm still working on, see the TODO.md and CHANGELOG.md files.
 
 ## Installation
@@ -43,6 +57,12 @@ After this command finishes (which might take a long time depending on what cate
 
 Now that you have a set of lyrics, it's time to convert them into something that Enter Rockstar can use.
 
+```
+$ enter-rockstar tokenize power_metal lyrics/
+```
+
+Depending on the amount of lyrics you feed into this command, this can take a lot of time. You should also be aware that lyrics that aren't in English will be skipped, as Rockstar isn't really supporting other human languages right now.
+
 ## Generating lyrics
 
 ### Finding words for poetic literals
@@ -50,10 +70,10 @@ Now that you have a set of lyrics, it's time to convert them into something that
 The most common and basic function of Enter Rockstar is just finding interesting words of good length to use in the poetic numeral representations. Finding out what words to use to represent `123` is not as easy as it might sound and this makes it easier:
 
 ```
-$ enter-rockstar poetic 123
+$ enter-rockstar poetic 123 lyrics_data/power_metal_tokens.json --amount 10
 ```
 
-
+The second argument should be a json tokens list generated in a previous step. You can also skip it, at which point Enter Rockstar will use its built-in list generated from Heavy Metal category.
 
 ## Contributing
 

data/TODO.md

@@ -0,0 +1,26 @@
+# 0.2
+
+Features:
+
+- [x] CLI for scraping lyrics and parsing them to JSON datasets
+- [x] Initial dataset from Heavy Metal lyrics included in the gem
+- [x] Random strategy and CLI for generating poetic representations of numbers
+- [x] Make sure that the generator works with both Floats and Integers
+- [x] Gzip/Gunzip data JSONs for smaller file sizes
+- [ ] Generator should fall back on bundled token set
+
+Other:
+
+- [x] Better progress indicator for parsing files
+- [ ] Test the wikia scraper fully
+- [x] Test generating poetic representations
+- [ ] Test CLI
+- [x] Fix tests on Ruby 2.3 and 2.4
+- [ ] Make sure that the basic tokens gzip is distributed with the gem so it's actually useful without spending
+      a day on scraping wikia
+
+# Future improvements
+
+- [ ] Add better generation strategies so the generated numbers potentially make more sense
+- [ ] Add a complex generator that generates variable names together with numbers
+- [ ] Add a translator so you can write minimal Rockstar and translate it automatically to proper Rockstar code

data/enter-rockstar.gemspec

@@ -24,13 +24,16 @@ Gem::Specification.new do |spec|
   spec.required_ruby_version = '>= 2.3'
 
   spec.add_development_dependency 'bundler', '~> 2.0'
+  spec.add_development_dependency 'fakefs'
   spec.add_development_dependency 'pry'
   spec.add_development_dependency 'rake'
   spec.add_development_dependency 'rspec'
   spec.add_development_dependency 'rubocop'
   spec.add_development_dependency 'vcr'
   spec.add_development_dependency 'webmock'
-  # spec.add_dependency 'hashie'
-  spec.add_dependency 'thor'
+
   spec.add_dependency 'nokogiri'
+  spec.add_dependency 'progressbar'
+  spec.add_dependency 'thor'
+  spec.add_dependency 'whatlanguage'
 end

data/lib/enter_rockstar/cli.rb

@@ -11,13 +11,13 @@ module EnterRockstar
     desc 'scrape_category CATEGORY_NAME URL', 'scrape lyrics wikia category page for bands and albums'
     def scrape_category(category_name, url)
       scraper = EnterRockstar::Scraper::Wikia.new(category_name: category_name, url: url)
-      scraper.parse_category(test_limit: true)
+      scraper.parse_category
       scraper.save_category
       say
     end
 
     desc 'scrape_lyrics CATEGORY_NAME START_INDEX', 'scrape actual lyrics from the lyrics wikia using the generated json file'
-    def scrape_lyrics(category_name, start_index=0)
+    def scrape_lyrics(category_name, start_index = 0)
       scraper = EnterRockstar::Scraper::Wikia.new(category_name: category_name)
       scraper.load_saved_json
       scraper.parse_all_pages(start_index: start_index)
@@ -30,18 +30,24 @@ module EnterRockstar
       scraper.print_indexed_tree
     end
 
-    desc 'tokenize DATA_DIR', 'take the downloaded lyrics text files and tokenize them'
-    def tokenize(data_dir)
-      tokenizer = EnterRockstar::Corpus::Tokenizer.new(data_dir: data_dir)
+    desc 'tokenize NAME DATA_DIR', 'take the downloaded lyrics text files and tokenize them'
+    def tokenize(name, data_dir)
+      tokenizer = EnterRockstar::Corpus::Tokenizer.new(data_dir: data_dir, name: name)
       tokenizer.tokenize
+      tokenizer.save_all
     end
 
-    desc 'poetic NUMBER', 'generate a poetic representation of a number from the word base'
-    def poetic(number)
-      generator = EnterRockstar::Generator::Poetic.new
-      results = generator.number(123)
+    desc 'poetic NUMBER SOURCE_JSON', 'generate a poetic representation of a number from the word base'
+    option :amount, desc: 'how many number representations should be generated'
+    option :strategy, desc: 'generating strategy. One of: [random weighted]'
+    def poetic(number, source_json)
+      amount = options[:amount] || 5
+      strategy = options[:strategy] || 'random'
 
-      say results
+      generator = EnterRockstar::Generator::Poetic.new(data_file: source_json, amount: amount, strategy: strategy)
+      results = generator.number(number)
+
+      say results.join("\n")
     end
   end
 end

data/lib/enter_rockstar/corpus/tokenizer.rb

@@ -1,15 +1,66 @@
 # frozen_string_literal: true
 
+require 'whatlanguage'
+require 'ruby-progressbar'
+
 module EnterRockstar
   module Corpus
     # take the downloaded lyrics texts and tokenize them
     class Tokenizer
-      def initialize(data_dir: )
+      def initialize(data_dir:, name:)
         @data_dir = data_dir
+        @stats = {}
+        @tokens = {}
+        @output_stats = "lyrics_data/#{name}_stats.json.gz"
+        @output_tokens = "lyrics_data/#{name}_tokens.json.gz"
+        @wl = WhatLanguage.new(:all)
       end
 
       def tokenize
+        text_files = Dir.glob("#{@data_dir}/**/*.txt")
+        puts "Parsing #{text_files.count} files."
+        progressbar = ProgressBar.create(title: 'Progress', total: text_files.count)
+
+        text_files.each do |filename|
+          # read the lyrics and tokenize the words
+          text = IO.read(filename)
+
+          # Rockstar doesn't really work well with languages other than English
+          if @wl.language(text) == :english
+            tokenized = _to_tokens(text)
+            # save stats which word appears after which one
+            n = 3
+            tokenized.each_cons(n) do |*head, continuation|
+              @stats[head] ||= Hash.new(0)
+
+              @stats[head][continuation] += 1
+            end
+
+            # save the words themselves based on what length they are
+            tokenized.each do |token|
+              next if token.length < 4 # shorter words are boring anyway
+
+              @tokens[token.length] ||= []
+              @tokens[token.length].push token unless @tokens[token.length].include? token
+            end
+            progressbar.increment
+          else
+            progressbar.increment
+            next
+          end
+        end
+        puts
+      end
+
+      def save_all
+        EnterRockstar::Utils.save_file(@output_tokens, @tokens.to_json)
+        EnterRockstar::Utils.save_file(@output_stats, @stats.to_json)
+      end
+
+      private
 
+      def _to_tokens(text)
+        text.downcase.split(/[^[[:alpha:]]]+/).reject(&:empty?).map(&:to_sym)
       end
     end
   end

data/lib/enter_rockstar/generator/poetic.rb

@@ -2,14 +2,49 @@
 
 module EnterRockstar
   module Generator
-    # lyrics scraper for lyrics.wikia.com
+    # poetic number generator
     class Poetic
-      def initialize
+      STRATEGIES = {
+        'random' => '_random'
+      }.freeze
 
+      attr_reader :tokens, :amount, :strategy
+
+      def initialize(data_file:, amount: 5, strategy: 'random')
+        @tokens = JSON.parse EnterRockstar::Utils.load_json(data_file)
+        @amount = Integer(amount) rescue 5
+        @strategy = STRATEGIES[strategy] || '_random'
       end
 
-      def number(number)
+      def number(num)
+        # split the number into parts
+        array = num.to_s.split(/\B|\b/)
+
+        all_results = []
+        @amount.times do
+          result = send(@strategy, array)
+          all_results.push result.join(' ')
+        end
+
+        all_results
+      end
+
+      private
+
+      def _random(array)
+        result = []
+        array.each do |digit|
+          if digit == '.'
+            result << '.'
+            next
+          end
+
+          # digits less than 4 should use longer words
+          digit = digit.to_i < 4 ? (digit.to_i + 10).to_s : digit
+          result << @tokens[digit].sample
+        end
 
+        result
       end
     end
   end

data/lib/enter_rockstar/scraper/wikia.rb

@@ -2,7 +2,6 @@
 
 require 'open-uri'
 require 'nokogiri'
-require 'json'
 
 module EnterRockstar
   module Scraper
@@ -12,18 +11,18 @@ module EnterRockstar
       DATA_DIR = 'lyrics'
       SLEEP_BETWEEN_REQUESTS = 0.1
 
-      attr_reader :tree
+      attr_reader :tree, :url, :category_name, :output
 
       def initialize(category_name: 'heavy_metal', url: '/wiki/Category:Genre/Heavy_Metal', data_dir: 'lyrics_data')
         @tree = {}
-        @output = "#{data_dir}/wikia_#{category_name}.json"
+        @output = "#{data_dir}/wikia_#{category_name}.json.gz"
         @url = url
         @category_name = category_name
       end
 
       def parse_category(url: nil, test_limit: false)
         url ||= START_HOST + @url
-        html = URI.open(url)
+        html = URI.parse(url).open
         doc = Nokogiri::HTML(html)
 
         # get all category member links and sort them by band and album
@@ -49,17 +48,11 @@ module EnterRockstar
       end
 
       def save_category
-        puts
-        out = File.new(@output, 'w')
-        out.write @tree.to_json
-        out.close
-        puts "Saved JSON data to #{@output}"
+        EnterRockstar::Utils.save_file(@output, @tree.to_json)
       end
 
       def load_saved_json
-        file = File.read(@output)
-        @tree = JSON.parse(file)
-        @new_tree = JSON.parse(file)
+        @tree = JSON.parse(EnterRockstar::Utils.load_json(@output))
       end
 
       def print_indexed_tree
@@ -78,24 +71,23 @@ module EnterRockstar
             dirname = k == 'band_url' ? [DATA_DIR, @category_name, key].join('/') : [DATA_DIR, @category_name, key, k].join('/')
             FileUtils.mkdir_p dirname
 
-            parse_page(v, dirname, key)
+            parse_page(v, dirname)
           end
         end
-
-        @tree = @new_tree
-        save_category
       end
 
-      def parse_page(url, dirname, band)
-        puts url
+      def parse_page(url, dirname)
         sleep SLEEP_BETWEEN_REQUESTS
-        html = URI.open(START_HOST + url)
+        html = URI.parse(START_HOST + url).open
         doc = Nokogiri::HTML(html)
 
         if doc.css('h2 span.mw-headline a').count.zero?
           # single album page listed on the category
           doc.css('div.mw-content-text ol li a').each do |song|
-            parse_song(song.attr('href'), dirname, song.text) if song&.attr('href')
+            next unless song&.attr('href')
+
+            lyrics = parse_song(song.attr('href'), dirname, song.text)
+            save_song("#{dirname}/#{song.text}.txt", lyrics) unless lyrics.nil?
           end
           puts
         else
@@ -104,11 +96,13 @@ module EnterRockstar
             # some band pages have extra albums that are not listed in the category page for some reason
             album_dirname = [dirname, album.text].join('/')
             FileUtils.mkdir_p album_dirname
-            @new_tree[band][album.text] = album.attr('href')
 
             # get song pages
             album.parent.parent.css('+ div + ol > li a').each do |song|
-              parse_song(song.attr('href'), album_dirname, song.text) if song&.attr('href')
+              next unless song&.attr('href')
+
+              lyrics = parse_song(song.attr('href'), album_dirname, song.text)
+              save_song("#{album_dirname}/#{song.text}.txt", lyrics) unless lyrics.nil?
             end
             puts
           end
@@ -126,21 +120,18 @@ module EnterRockstar
 
         print '.'
         sleep SLEEP_BETWEEN_REQUESTS
-        html = URI.open(START_HOST + url)
+        html = URI.parse(START_HOST + url).open
         doc = Nokogiri::HTML(html)
 
         lyrics = doc.css('div.lyricbox').first
         return if lyrics.nil?
+        return if lyrics.css('a')&.first&.attr('href') == '/wiki/Category:Instrumental'
 
-        if lyrics.css('a')&.first&.attr('href') == '/wiki/Category:Instrumental'
-          # instrumental song, whatever
-        else
-          proper_text = lyrics.inner_html.gsub(%r{<div.*?(\/div>)}, '').split('<br>').join("\n")
+        lyrics.inner_html.split('<br>').join("\n").gsub(%r{<\/?[^>]*>}, '')
+      end
 
-          out = File.new(songfile, 'w')
-          out.write proper_text
-          out.close
-        end
+      def save_song(songfile, contents)
+        EnterRockstar::Utils.save_plain(songfile, contents)
       end
     end
   end

data/lib/enter_rockstar/utils.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require 'zlib'
+
+module EnterRockstar
+  # shared utility code for different modules
+  class Utils
+    def self.load_json(file)
+      if File.exist?(file) && file.end_with?('.gz')
+        data = Zlib::GzipReader.new(StringIO.new(IO.read(file))).read
+      elsif File.exist? file.sub('.gz', '')
+        data = IO.read(file.sub('.gz', ''))
+      else
+        raise IOError, "File not found: #{file}"
+      end
+
+      data
+    end
+
+    def self.save_file(filename, contents)
+      File.open(filename, 'w') do |f|
+        gz = Zlib::GzipWriter.new(f)
+        gz.write contents
+        gz.close
+      end
+      puts "Saved as #{filename}"
+    end
+
+    def self.save_plain(filename, contents)
+      File.open(filename, 'w') do |f|
+        f.write contents
+      end
+      puts "Saved as #{filename}"
+    end
+  end
+end

data/lib/enter_rockstar/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module EnterRockstar
-  VERSION = '0.1'
+  VERSION = '0.2'
 end

data/lib/enter_rockstar.rb

@@ -1,6 +1,12 @@
 # frozen_string_literal: true
 
+require 'json'
+require 'zlib'
+
+require 'enter_rockstar/utils'
 require 'enter_rockstar/scraper/wikia'
+require 'enter_rockstar/corpus/tokenizer'
+require 'enter_rockstar/generator/poetic'
 require 'pry'
 
 module EnterRockstar