emoji_sub 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a1f89458d20fe54dd6b7b2a20840cec05f7d4584eae58c0756c0a8c78f116b9d
-  data.tar.gz: 83490523c30b2dfe4afdd6c36822bdf468e9b510efd9dc0c976c7167c71dfeca
+  metadata.gz: 7de8cfebd03d2b8fcb5200d0d22d7ba24da54cdb1a718cf5fe101ba40a5671ea
+  data.tar.gz: f2069d0428b38d59ddd9a05d5ae597472c31dcdd3a7572a4886c35e6aadae430
 SHA512:
-  metadata.gz: d6b6501f9b5c4ceb2b77a9f73d2b53f18450c4e063e5e2b44cc27e69db4aeeeff336579af21b202bfd2560e7e6cb40f7f48fac6009ef19abc54b07cf1330dc82
-  data.tar.gz: 44b10917c640e016c165fb87370a64d399207770ebd7e19d22dc1270aa61cdce63ba222e301a5b2fc866bf7d8cffabec6bf458027934f6c2714f0c9b7a884a87
+  metadata.gz: deeeb7b86c4eacf419a72ecf7594cc377fe0dce08206946b698951264363b6f8166d8a21facba636b0d975ab552303d31ca3dca371f3112e27a8141354cd3462
+  data.tar.gz: 1be05961c5b9bed283a36b149a9bf92dfbd54ba7000ca5333d4b2b6d3e2c38cef07f86ad0ada0b2b9a1c82d9a400fb8c314ef2173cae2b11fac26cd861caf780

data/CHANGELOG.md

@@ -4,6 +4,15 @@ All notable changes to this project will be documented in this file.
 
 This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.2.0] - 2020-08-22
+
+Last night I wrote a script to scrape down all the emoji definitions w/ shortcodes from emojipedia. I refined it, added specs, and created a rake task for it.
+
+### Added
+
+- `EmojiScrapr` (available for development use only)
+- specs
+
 ## [0.1.0] - 2020-08-21
 
 This is the first release. The goal is to get a method that works for substitution of
@@ -15,4 +24,5 @@ emoji HTML entities (hex-unicode) for slack shortcut format (:this_kind:)
 - data/emoji.yml file, 
 - Specs covering basic funcitonality
 
-[0.1.0]: https://github.com/olivierlacan/keep-a-changelog/releases/tag/v0.1.0
+[0.2.0]: https://github.com/armahillo/emoji_sub/compare/v0.1.0...v0.2.0
+[0.1.0]: https://github.com/armahillo/emoji_sub/releases/tag/v0.1.0

data/Gemfile

@@ -5,3 +5,9 @@ gemspec
 
 gem "rake", "~> 12.0"
 gem "rspec", "~> 3.0"
+
+# For EmojiScrapr
+gem 'mechanize' 
+
+# For testing with mechanize. The @chrisk fork updates it
+gem 'fakeweb', git: 'https://github.com/chrisk/fakeweb.git' 

data/Gemfile.lock

@@ -1,12 +1,42 @@
+GIT
+  remote: https://github.com/chrisk/fakeweb.git
+  revision: 2b08c1ff2714ec13a12f3497d67fcefce95c2cbe
+  specs:
+    fakeweb (1.3.0)
+
 PATH
   remote: .
   specs:
-    emoji_sub (0.1.0)
+    emoji_sub (0.2.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
+    connection_pool (2.2.3)
     diff-lcs (1.4.4)
+    domain_name (0.5.20190701)
+      unf (>= 0.0.5, < 1.0.0)
+    http-cookie (1.0.3)
+      domain_name (~> 0.5)
+    mechanize (2.7.6)
+      domain_name (~> 0.5, >= 0.5.1)
+      http-cookie (~> 1.0)
+      mime-types (>= 1.17.2)
+      net-http-digest_auth (~> 1.1, >= 1.1.1)
+      net-http-persistent (>= 2.5.2)
+      nokogiri (~> 1.6)
+      ntlm-http (~> 0.1, >= 0.1.1)
+      webrobots (>= 0.0.9, < 0.2)
+    mime-types (3.3.1)
+      mime-types-data (~> 3.2015)
+    mime-types-data (3.2020.0512)
+    mini_portile2 (2.4.0)
+    net-http-digest_auth (1.4.1)
+    net-http-persistent (4.0.0)
+      connection_pool (~> 2.2)
+    nokogiri (1.10.10)
+      mini_portile2 (~> 2.4.0)
+    ntlm-http (0.1.1)
     rake (12.3.3)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)
@@ -21,12 +51,18 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.9.0)
     rspec-support (3.9.3)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.7.7)
+    webrobots (0.1.2)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
   emoji_sub!
+  fakeweb!
+  mechanize
   rake (~> 12.0)
   rspec (~> 3.0)
 

data/Rakefile

@@ -4,3 +4,31 @@ require "rspec/core/rake_task"
 RSpec::Core::RakeTask.new(:spec)
 
 task :default => :spec
+
+namespace :scrape do
+
+  desc "Pull down all emoji into YML files (long execution: 3-4 hours)"
+  task :all do
+    require 'development/emoji_scrapr'
+    destination_dir = '/tmp/'
+    scrapr = EmojiScrapr.new
+    duration = scrapr.scrape_all
+  ensure
+    puts "Done! #{duration} secs"
+    puts "Saved to: " + scrapr.save
+  end
+
+  def harvest(destination = DUMP_PATH)
+    @categories.each do |section, url|
+      @all_emoji[section] = collect_emoji_list(url)
+      dump_to_yaml(section + '_urls', scrapr.all_emoji.fetch(section,[]))
+      collect_all_emoji(section)
+      dump_to_yaml(section, scrapr.all_emoji.fetch(section,[]))
+    end
+  end
+
+  def collect_all_emoji(section , delay = 2)
+    
+    scrape_all_emoji_in_category(@all_emoji[section], delay)
+  end
+end

data/emoji_sub.gemspec

@@ -19,7 +19,7 @@ Gem::Specification.new do |spec|
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
   spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
-    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec|lib\/development)/}) }
   end
   spec.bindir        = "exe"
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }

data/lib/emoji_sub.rb

@@ -1,5 +1,5 @@
 require "emoji_sub/version"
-require 'YAML'
+require 'yaml'
 
 module EmojiSub
   EMOJI_MAPPING_YAML = "data/emoji.yaml"

data/lib/emoji_sub/version.rb

@@ -1,3 +1,3 @@
 module EmojiSub
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: emoji_sub
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Aaron Hill