emojidex 0.0.23 → 0.1.0

data/lib/emojidex/data/collection/cache.rb

@@ -0,0 +1,137 @@
+require 'json'
+require 'fileutils'
+require_relative 'asset_information'
+require_relative '../../service/transactor'
+require_relative '../../defaults'
+
+module Emojidex
+  module Data
+    # local caching functionality for collections
+    module CollectionCache
+      include Emojidex::Data::CollectionAssetInformation
+      attr_reader :cache_path, :download_queue
+      attr_accessor :download_threads
+
+      def setup_cache(path = nil)
+        @download_queue = []
+        @download_threads = 8
+        # check if cache dir is already set
+        return @cache_path if @cache_path && path.nil?
+        # setup cache
+        @cache_path = File.expand_path((path || ENV['EMOJI_CACHE'] || "#{ENV['HOME']}/.emojidex/") + '/emoji')
+        # ENV['EMOJI_CACHE'] = @cache_path
+        FileUtils.mkdir_p(@cache_path)
+        Emojidex::Defaults.sizes.keys.each do |size|
+          FileUtils.mkdir_p(@cache_path + "/#{size}")
+        end
+        @cache_path
+      end
+
+      # Caches emoji to local emoji storage cache
+      # Options:
+      #   cache_path: manually specify cache location
+      #     (default is ENV['EMOJI_CACHE'] or '$HOME/.emoji_cache')
+      #   formats: formats to cache (default is SVG only)
+      #   sizes: sizes to cache (default is px32, but this is irrelivant for SVG)
+      def cache!(options = {})
+        setup_cache options[:cache_path]
+        formats = options[:formats] || Emojidex::Defaults.selected_formats
+        sizes = options[:sizes] || Emojidex::Defaults.selected_sizes
+        @emoji.values.each do |moji|
+          _svg_check_copy(moji) if formats.include? :svg
+          _raster_check_copy(moji, :png, sizes) if formats.include? :png
+        end
+        _process_download_queue
+        cache_index
+      end
+
+      # Updates an index in the specified destination (or the cache path if not specified).
+      # This method reads the existing index, combines the contents with this collection, and
+      # writes the results.
+      def cache_index(destination = nil)
+        destination ||= @cache_path
+        idx = Emojidex::Data::Collection.new
+        idx.load_local_collection(destination) if FileTest.exist? "#{destination}/emoji.json"
+        idx.add_emoji @emoji.values
+        File.open("#{destination}/emoji.json", 'w') { |f| f.write idx.emoji.values.to_json }
+      end
+
+      # [over]writes a sanitized index to the specified destination.
+      # WARNING: This method destroys any index files in the destination.
+      def write_index(destination)
+        idx = @emoji.values.to_json
+        idx = JSON.parse idx
+        idx.each { |moji| moji.delete_if{ |k, v| v.nil? }}
+        File.open("#{destination}/emoji.json", 'w') { |f| f.write idx.to_json }
+      end
+
+      private
+
+      def _svg_check_copy(moji)
+        @download_queue << { moji: moji, formats: :svg, sizes: [] } if @vector_source_path.nil? && @source_path.nil?
+        @vector_source_path = @source_path if @vector_source_path.nil?
+        src = "#{@vector_source_path}/#{moji.code}.svg"
+        if File.exist? "#{src}"
+          unless File.exist?("#{@cache_path}/#{moji.code}") &&
+              FileUtils.compare_file("#{src}", "#{@cache_path}/#{moji.code}.svg")
+            FileUtils.cp("#{src}", @cache_path)
+          end
+        else
+          _cache_svg_from_net(moji)
+        end
+        FileUtils.cp_r src, @cache_path if File.directory? src
+      end
+
+      def _raster_check_copy(moji, format, sizes)
+        @download_queue << { moji: moji, formats: [format], sizes: sizes } if @raster_source_path.nil? && @source_path.nil?
+        @raster_source_path = @source_path if @raster_source_path.nil?
+        _cache_raster_from_net(moji, format, sizes) if @raster_source_path.nil?
+        sizes.each do |size|
+          src = "#{@raster_source_path}/#{size}/#{moji.code}"
+          if FileTest.exist? "#{src}.#{format}"
+            FileUtils.cp("#{src}.#{format}", ("#{@cache_path}/#{size}"))
+          else 
+            _cache_raster_from_net(moji, format, sizes)
+          end
+          FileUtils.cp_r(src, @cache_path) if File.directory? src
+        end
+      end
+
+      def _process_download_queue
+        thr = []
+        @download_queue.each do |dl|
+          thr << Thread.new { _cache_from_net(dl[:moji], dl[:formats], dl[:sizes]) }
+          thr.each { |t| t.join } if thr.length >= @download_threads
+        end
+      end
+
+      def _cache_from_net(moji, formats, sizes)
+        formats = *formats unless formats.class == Array
+        dls = []
+        dls << Thread.new { _cache_svg_from_net(moji) } if formats.include? :svg
+        dls << Thread.new { _cache_raster_from_net(moji, :png, sizes) } if formats.include? :png
+        dls.each { |t| t.join }
+      end
+
+      def _cache_svg_from_net(moji)
+        target = "#{@cache_path}/#{moji.code}.svg"
+        if File.exist? target # check for an existing copy so we don't double downlaod
+          return if moji.checksum?(:svg).nil? # no updates if we didn't get details
+          # if the checksums are the same there is no reason to update
+          return if moji.checksum?(:svg) == get_checksums(moji, [:svg])[:svg]
+        end
+        response = Emojidex::Service::Transactor.download("#{moji.code}.svg")
+        File.open(target, 'wb') { |fp| 
+          fp.write(response.body) }
+      end
+
+      def _cache_raster_from_net(moji, format, sizes)
+        sizes.each do |size|
+          response = Emojidex::Service::Transactor.download("#{size}/#{moji.code}.#{format.to_s}")
+          File.open("#{@cache_path}/#{size}/#{moji.code}.#{format.to_s}", 'wb') { |fp| 
+            fp.write(response.body) }
+        end
+      end
+    end
+  end
+end

data/lib/emojidex/{collection → data/collection}/moji_data.rb

@@ -1,4 +1,4 @@
-module Emojidex
+module Emojidex::Data
   # collects and condenses UTF moji codes within a collection
   module CollectionMojiData
     attr_reader :moji_code_string, :moji_code_index

data/lib/emojidex/data/collection/static_collection.rb

@@ -0,0 +1,35 @@
+require_relative '../../service/transactor'
+require_relative '../../env_helper'
+
+module Emojidex
+  module Data
+    # mixin module to enable static collections
+    module StaticCollection
+      def load_from_server(detailed = true, locale = '??')
+        locale = Emojidex::EnvHelper.lang? if locale == '??'
+        begin
+          res = Emojidex::Service::Transactor.get(@endpoint, {detailed: detailed, locale: locale})
+        rescue
+          return false
+        end
+        add_emoji(res)
+        return true
+      end
+
+      def check_and_load_static(collection)
+        loaded = false
+        if defined? Emojidex::Vectors
+          @vector_source_path = Emojidex::Vectors.path + "/#{collection}/"
+          load_local_collection @vector_source_path
+          loaded = true
+        end
+        if defined? Emojidex::Rasters
+          @raster_source_path = Emojidex::Rasters.path + "/#{collection}/"
+          load_local_collection @raster_source_path
+          loaded = true
+        end
+        loaded
+      end
+    end
+  end
+end

data/lib/emojidex/data/collection_checker.rb

@@ -0,0 +1,94 @@
+require 'find'
+require_relative '../defaults.rb'
+
+module Emojidex
+  module Data
+    # Check collections for presence of image assets and discrepencies in emoji indexes.
+    class CollectionChecker
+      attr_reader :index_only, :asset_only
+
+      def initialize(collections, options = {})
+        collections = *collections
+        @index_only = {}
+        @asset_only = {}
+        asset_path = options[:asset_path] || collections.first.source_path
+        sizes = options[:sizes] || Emojidex::Defaults.sizes.keys
+        formats = options[:formats] || Emojidex::Defaults.formats
+
+        asset_files = create_asset_file_list(asset_path, sizes, formats)
+        check_for_index_only(collections, asset_files, sizes, formats)
+        check_for_asset_only(collections, asset_files)
+      end
+
+      private
+
+      def create_asset_file_list(dir, sizes, formats)
+        result = {}
+        result.merge!(create_file_list(dir, '.svg', '')) if formats.include?(:svg)
+        if formats.include?(:png)
+          sizes.each do |size|
+            result.merge!(create_file_list("#{dir}/#{size}", '.png', "#{size}/"))
+          end
+        end
+        result
+      end
+
+      def create_file_list(dir, ext, prefix)
+        result = {}
+        return result unless Dir.exists?(dir)
+        Dir.foreach(dir) do |file|
+          result["#{prefix}#{File.basename(file, '.*')}".to_sym] =
+            "#{prefix}#{file}" if File.extname(file) == ext
+        end
+        result
+      end
+
+      def check_for_index_only(collections, asset_files, sizes, formats)
+        collections.each do |collection|
+          collection.emoji.values.each do |emoji|
+            tmp = []
+            tmp += create_svg_array(emoji, asset_files) if formats.include?(:svg)
+            tmp += create_png_array(emoji, asset_files, sizes) if formats.include?(:png)
+            @index_only[emoji.code.to_sym] = tmp unless tmp.empty?
+          end
+        end
+      end
+
+      def create_svg_array(emoji, asset_files)
+        result = []
+        asset_file = asset_files[emoji.code.to_sym]
+        result << "#{emoji.code}.svg" if asset_files[emoji.code.to_sym].nil?
+        result
+      end
+
+      def create_png_array(emoji, asset_files, sizes)
+        result = []
+        sizes.each do |size|
+          name = "#{size}/#{emoji.code}"
+          result << "#{name}.png" if asset_files[name.to_sym].nil?
+        end
+        result
+      end
+
+      def check_for_asset_only(collections, asset_files)
+        asset_files.each do |_key, value|
+          code = File.basename(value, '.*')
+
+          next if find_emoji_from_collections(collections, code)
+
+          symbol = code.to_sym
+          @asset_only[symbol] = [] if @asset_only[symbol].nil?
+          @asset_only[symbol] << value
+        end
+      end
+
+      def find_emoji_from_collections(collections, code)
+        symbol = code.to_sym
+        collections.each do |collection|
+          return true unless collection.emoji[symbol].nil?
+        end
+        false
+      end
+    end
+  end
+end

data/lib/emojidex/{emoji.rb → data/emoji.rb}

@@ -1,14 +1,15 @@
 require_relative 'emoji/asset_information'
 
-module Emojidex
+module Emojidex::Data
   # emoji base class
   class Emoji
     attr_accessor :moji, :category, :code, :code_ja,
                   :unicode, :tags, :emoticon, :variants, :base
 
-    include Emojidex::EmojiAssetInformation
+    include Emojidex::Data::EmojiAssetInformation
 
     def initialize(details = {})
+      init_asset_info(details)
       @moji = details[:moji]
       @code, @code_ja = details[:code], details[:code_ja]
       @unicode, @full_name = details[:unicode], details[:full_name]

data/lib/emojidex/data/emoji/asset_information.rb

@@ -0,0 +1,45 @@
+require_relative '../../defaults'
+
+module Emojidex
+  module Data
+    # Asset information for emoji
+    module EmojiAssetInformation
+      attr_accessor :checksums, :paths
+
+      def init_asset_info(details)
+        blank_checksums
+        fill_checksums(details[:checksums]) if details.include? :checksums
+      end
+
+      # returns asset checksum
+      def checksum?(format, size = nil)
+        puts @checksums
+        return @checksums[format][size] unless size.nil?
+        @checksums[format]
+      end
+
+      def blank_checksums
+        @checksums = {}
+        @checksums[:svg] = nil
+        @checksums[:png] = {}
+        Emojidex::Defaults.sizes.keys.each do |size|
+          @checksums[:png][size] = nil
+        end
+      end
+
+      def fill_checksums(checksums)
+        @checksums[:svg] = checksums[:svg] if checksums.include? :svg
+        return unless checksums.include? :png
+        Emojidex::Defaults.sizes.keys.each do |size|
+          @checksums[:png][size] = checksums[:png][size] if checksums[:png].include? size
+        end
+      end
+
+      # returns asset path
+      def path?(format, size = nil)
+        return @paths[format][size] unless size.nil?
+        @paths[format]
+      end
+    end
+  end
+end

data/lib/emojidex/data/extended.rb

@@ -0,0 +1,18 @@
+require_relative 'collection'
+require_relative 'collection/static_collection'
+
+module Emojidex
+  module Data
+    # listing and search of extended emoji from the emojidex set
+    class Extended < Collection
+      include Emojidex::Data::StaticCollection
+
+      def initialize
+        super
+        @endpoint = 'extended_emoji'
+        load_from_server unless check_and_load_static('extended')
+        @emoji
+      end
+    end
+  end
+end

data/lib/emojidex/data/utf.rb

@@ -0,0 +1,18 @@
+require_relative 'collection'
+require_relative 'collection/static_collection'
+
+module Emojidex
+  module Data
+    # listing and search of standard UTF emoji
+    class UTF < Collection
+      include Emojidex::Data::StaticCollection
+
+      def initialize
+        super
+        @endpoint = 'utf_emoji'
+        load_from_server unless check_and_load_static('utf')
+        @emoji
+      end
+    end
+  end
+end

data/lib/emojidex/defaults.rb

@@ -1,14 +1,35 @@
 module Emojidex
   # Global defines for emojidex
   class Defaults
+    @@selected_sizes = [:mdpi, :px32]
+    @@selected_formats = [:png]
+
     def self.sizes
       { ldpi: 13, mdpi: 18, hdpi: 27, xhdpi: 36, xxhdpi: 54, xxxhdpi: 72,
         px8: 8, px16: 16, px32: 32, px64: 64, px128: 128, px256: 256, px512: 512,
         hanko: 90, seal: 320 }
     end
 
+    def self.selected_sizes(sizes = nil)
+      @@selected_sizes = sizes unless sizes.nil?
+      @@selected_sizes
+    end
+
     def self.formats
       [:svg, :png]
     end
+
+    def self.selected_formats(formats = nil)
+      @@selected_formats = formats unless formats.nil?
+      @@selected_formats
+    end
+
+    def self.limit
+      50
+    end
+
+    def self.lang
+      'en'
+    end
   end
 end

data/lib/emojidex/env_helper.rb

@@ -0,0 +1,11 @@
+module Emojidex
+  # Obtains and parses local environment variables
+  module EnvHelper
+
+    def self.lang?
+      lang = ENV['LANG'].match('^..')
+      lang = 'en' unless lang.to_s == 'ja'
+      lang
+    end
+  end
+end

data/lib/emojidex/service/collection.rb

@@ -0,0 +1,67 @@
+require_relative '../data/collection'
+require_relative 'transactor'
+require_relative '../defaults'
+
+module Emojidex
+  module Service
+    # A modified collection class for collections tied to the emojidex service
+    class Collection < Emojidex::Data::Collection
+      attr_reader :endpoint, :page, :limit, :detailed, :auto_cache
+
+      def initialize(opts = {})
+        @emoji = {}
+        add_emoji(opts[:emoji]) && opts.delete(:emoji) if opts.include? :emoji
+
+        @username = opts[:username] || nil
+        opts.delete(:username)
+        @auth_token = opts[:auth_token] || nil
+        opts.delete(:auth_token)
+
+        @endpoint = opts[:endpoint] || 'emoji'
+        opts.delete(:endpoint)
+        @page = opts[:page] || 0
+        opts.delete(:page)
+        @limit = opts[:limit] || Emojidex::Defaults.limit
+        opts.delete(:limit)
+        @detailed = opts[:detailed] || false
+        opts.delete(:detailed)
+
+        @auto_cache = opts[:auto_cache] || true
+        opts.delete(:auto_cache)
+
+        auto_init = opts[:auto_init] || true
+        opts.delete(:auto_init)
+
+        @opts = opts
+
+        more if auto_init
+        @emoji
+      end
+
+      # Get the next page worth of emoji and add them to the collection
+      def more()
+        @page += 1
+
+        opts = { page: @page, limit: @limit, detailed: @detailed }
+        opts[:username] = @username unless @username.nil?
+        opts[:auth_token] = @auth_token unless @auth_token.nil?
+        opts.merge! @opts
+
+        page_moji = Emojidex::Service::Transactor.get(@endpoint, opts)
+
+        if page_moji.is_a? Hash
+          unless page_moji.key? :emoji
+            @page -= 1 # reset page beacuse we failed
+            return {}
+          end
+
+          add_emoji(page_moji[:emoji])
+          return page_moji[:emoji]
+        end
+        add_emoji(page_moji)
+        cache! if @auto_cache
+        page_moji
+      end
+    end
+  end
+end