emojidex 0.2.3 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/emojidex.gemspec +2 -1
- data/lib/emojidex/data/collection/cache.rb +26 -22
- data/lib/emojidex/data/emoji/asset_information.rb +30 -18
- data/lib/emojidex/service/collection.rb +15 -6
- data/lib/emojidex/service/transactor.rb +5 -3
- metadata +16 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 9204688c02e7450be481699ce1444f6b2966ee20
|
4
|
+
data.tar.gz: c398c6963a4e691f2bd64c10cd33a9fb943753c3
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e2f4beac1e132008e2c7ce68767f1eebfeabdb2899d95a8eec6d0c944972b907908407a3648a42fc9d010c69999bc7a1b278c82569df557a4ee917b6d61d42bf
|
7
|
+
data.tar.gz: d6a497d83bbcb0ce9886cafa3ccfea81af4b4eb518700283bd57b7d1a4699ed973fa735e605ac04effa2ac0d154bc667f92ba712b8c2e645d6432fc3f79e5ef4
|
data/emojidex.gemspec
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'emojidex'
|
3
|
-
s.version = '0.
|
3
|
+
s.version = '0.3.0'
|
4
4
|
s.license = 'emojiOL'
|
5
5
|
s.summary = 'emojidex Ruby tools'
|
6
6
|
s.description = 'emojidex emoji handling, search and lookup, listing and caching functionality' \
|
@@ -17,4 +17,5 @@ Gem::Specification.new do |s|
|
|
17
17
|
|
18
18
|
s.add_dependency 'faraday', '~> 0.9', '~> 0.9.2'
|
19
19
|
s.add_dependency 'faraday_middleware', '~> 0.10', '~> 0.10.0'
|
20
|
+
s.add_dependency 'typhoeus'
|
20
21
|
end
|
@@ -10,11 +10,13 @@ module Emojidex
|
|
10
10
|
module CollectionCache
|
11
11
|
include Emojidex::Data::CollectionAssetInformation
|
12
12
|
attr_reader :cache_path, :download_queue
|
13
|
-
attr_accessor :download_threads
|
13
|
+
attr_accessor :download_threads, :formats, :sizes
|
14
14
|
|
15
15
|
def setup_cache(path = nil)
|
16
|
+
@formats = Emojidex::Defaults.selected_formats unless @formats
|
17
|
+
@sizes = Emojidex::Defaults.selected_sizes unless @sizes
|
16
18
|
@download_queue = []
|
17
|
-
@download_threads =
|
19
|
+
@download_threads = 4
|
18
20
|
# check if cache dir is already set
|
19
21
|
return @cache_path if @cache_path && path.nil?
|
20
22
|
# setup cache
|
@@ -79,7 +81,7 @@ module Emojidex
|
|
79
81
|
idx = JSON.parse idx
|
80
82
|
idx.each do |moji|
|
81
83
|
moji['paths'] = nil
|
82
|
-
moji['
|
84
|
+
moji['remote_checksums'] = nil
|
83
85
|
moji.delete_if { |_k, v| v.nil? }
|
84
86
|
end
|
85
87
|
File.open("#{destination}/emoji.json", 'w') { |f| f.write idx.to_json }
|
@@ -88,24 +90,29 @@ module Emojidex
|
|
88
90
|
private
|
89
91
|
|
90
92
|
def _svg_check_copy(moji)
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
@
|
93
|
+
if @vector_source_path != nil
|
94
|
+
src_d = "#{@vector_source_path}/#{moji.code}"
|
95
|
+
src = "#{src_d}.svg"
|
96
|
+
|
97
|
+
FileUtils.cp(src, @cache_path) if File.exist?(src) && @vector_source_path != @cache_path
|
98
|
+
FileUtils.cp_r(src_d, @cache_path) if File.directory?(src_d) && @vector_source_path != @cache_path # Copies source frames and data files if unpacked
|
99
|
+
@download_queue << { moji: moji, formats: :svg, sizes: [] } unless File.exist?("#{@cache_path}/#{moji.code}.svg") # nothing was copied, get from net
|
100
|
+
return
|
96
101
|
end
|
97
|
-
|
102
|
+
|
103
|
+
@download_queue << { moji: moji, formats: :svg, sizes: [] }
|
98
104
|
end
|
99
105
|
|
100
106
|
def _raster_check_copy(moji, format, sizes)
|
101
107
|
sizes.each do |size|
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
108
|
+
if @raster_source_path != nil
|
109
|
+
src_d = "#{@raster_source_path}/#{size}/#{moji.code}"
|
110
|
+
src = "#{src_d}.#{format}"
|
111
|
+
|
112
|
+
FileUtils.cp("#{src}", ("#{@cache_path}/#{size}")) if File.exist?(src) && @raster_source_path != @cache_path
|
113
|
+
FileUtils.cp_r(src, @cache_path) if File.directory?(src_d) && @raster_source_path != @cache_path # Copies source frames and data files if unpacked
|
114
|
+
@download_queue << { moji: moji, formats: [format], sizes: [size] } unless File.exist?("#{@cache_path}/#{size}/#{moji.code}.#{format}") # nothing was copied, get from net
|
107
115
|
end
|
108
|
-
FileUtils.cp_r(src, @cache_path) if File.directory? src # Copies source frames and data files if unpacked
|
109
116
|
end
|
110
117
|
end
|
111
118
|
|
@@ -113,7 +120,7 @@ module Emojidex
|
|
113
120
|
thr = []
|
114
121
|
@download_queue.each do |dl|
|
115
122
|
thr << Thread.new { _cache_from_net(dl[:moji], dl[:formats], dl[:sizes]) }
|
116
|
-
thr.each(&:join) if thr.length >= @download_threads
|
123
|
+
thr.each(&:join) if thr.length >= (@download_threads - 1)
|
117
124
|
end
|
118
125
|
thr.each(&:join) # grab end of queue
|
119
126
|
end
|
@@ -128,12 +135,9 @@ module Emojidex
|
|
128
135
|
|
129
136
|
def _cache(options)
|
130
137
|
setup_cache options[:cache_path]
|
131
|
-
formats = options[:formats] ||
|
132
|
-
sizes = options[:sizes] ||
|
133
|
-
|
134
|
-
@vector_source_path = @cache_path if @vector_source_path.nil?
|
135
|
-
@raster_source_path = @cache_path if @raster_source_path.nil?
|
136
|
-
|
138
|
+
formats = options[:formats] || @formats
|
139
|
+
sizes = options[:sizes] || @sizes
|
140
|
+
|
137
141
|
@emoji.values.each do |moji|
|
138
142
|
_svg_check_copy(moji) if formats.include? :svg
|
139
143
|
_raster_check_copy(moji, :png, sizes) if formats.include? :png
|
@@ -5,11 +5,12 @@ module Emojidex
|
|
5
5
|
module Data
|
6
6
|
# Asset information for emoji
|
7
7
|
module EmojiAssetInformation
|
8
|
-
attr_accessor :checksums, :paths, :
|
8
|
+
attr_accessor :checksums, :paths, :remote_checksums
|
9
9
|
|
10
10
|
def init_asset_info(details)
|
11
|
+
blank_paths
|
11
12
|
blank_checksums
|
12
|
-
|
13
|
+
fill_remote_checksums(details[:checksums]) if details.include? :checksums
|
13
14
|
end
|
14
15
|
|
15
16
|
# returns asset checksum
|
@@ -26,7 +27,16 @@ module Emojidex
|
|
26
27
|
Emojidex::Defaults.sizes.keys.each do |size|
|
27
28
|
@checksums[:png][size] = nil
|
28
29
|
end
|
29
|
-
@
|
30
|
+
@remote_checksums = @checksums.dup
|
31
|
+
end
|
32
|
+
|
33
|
+
def fill_remote_checksums(checksums)
|
34
|
+
@remote_checksums[:svg] = checksums[:svg] if checksums.include? :svg
|
35
|
+
return unless checksums.include? :png
|
36
|
+
Emojidex::Defaults.sizes.keys.each do |size|
|
37
|
+
@remote_checksums[:png][size] = checksums[:png][size] if checksums[:png].include? size
|
38
|
+
end
|
39
|
+
@remote_checksums
|
30
40
|
end
|
31
41
|
|
32
42
|
def fill_checksums(checksums)
|
@@ -35,6 +45,7 @@ module Emojidex
|
|
35
45
|
Emojidex::Defaults.sizes.keys.each do |size|
|
36
46
|
@checksums[:png][size] = checksums[:png][size] if checksums[:png].include? size
|
37
47
|
end
|
48
|
+
@checksums
|
38
49
|
end
|
39
50
|
|
40
51
|
def blank_paths
|
@@ -52,6 +63,7 @@ module Emojidex
|
|
52
63
|
Emojidex::Defaults.sizes.keys.each do |size|
|
53
64
|
@paths[:png][size] = paths[:png][size] if paths[:png].include? size
|
54
65
|
end
|
66
|
+
@paths
|
55
67
|
end
|
56
68
|
|
57
69
|
# Acquires path and caches the target file if not found or out of date
|
@@ -75,7 +87,7 @@ module Emojidex
|
|
75
87
|
|
76
88
|
# Caches a file of the specified format in the specified sizes
|
77
89
|
def cache(format, sizes = nil)
|
78
|
-
|
90
|
+
generate_checksums
|
79
91
|
case format
|
80
92
|
when :png
|
81
93
|
_cache_png(sizes) unless sizes.nil?
|
@@ -85,33 +97,33 @@ module Emojidex
|
|
85
97
|
end
|
86
98
|
|
87
99
|
# Generates a checksum for each locally cached file
|
88
|
-
def
|
100
|
+
def generate_checksum(format, size = nil)
|
89
101
|
case format
|
90
102
|
when :png
|
91
|
-
return @
|
103
|
+
return @checksums[:png][size] = _checksum_for_file(@paths[:png][size])
|
92
104
|
when :svg
|
93
|
-
return @
|
105
|
+
return @checksums[:svg] = _checksum_for_file(@paths[:svg])
|
94
106
|
end
|
95
107
|
nil
|
96
108
|
end
|
97
109
|
|
98
|
-
def
|
99
|
-
@
|
110
|
+
def generate_checksums
|
111
|
+
@checksums[:svg] = _checksum_for_file(@paths[:svg])
|
100
112
|
@paths[:png].keys.each do |size|
|
101
|
-
@
|
113
|
+
@checksums[:png][size] = _checksum_for_file(@paths[:png][size])
|
102
114
|
end
|
103
|
-
@
|
115
|
+
@checksums
|
104
116
|
end
|
105
117
|
|
106
118
|
private
|
107
119
|
|
108
120
|
def _cache_svg
|
109
121
|
@paths[:svg] = Dir.pwd unless (@paths.include? :svg) && !@paths[:svg].nil?
|
110
|
-
return if File.exist?(@paths[:svg]) && !@
|
111
|
-
@
|
122
|
+
return if File.exist?(@paths[:svg]) && (!@remote_checksums[:svg].nil? &&
|
123
|
+
@remote_checksums[:svg] == generate_checksum(:svg))
|
112
124
|
response = Emojidex::Service::Transactor.download("#{@code}.svg")
|
113
125
|
File.open(@paths[:svg], 'wb') { |fp| fp.write(response.body) }
|
114
|
-
|
126
|
+
generate_checksum(:svg)
|
115
127
|
end
|
116
128
|
|
117
129
|
def _cache_png(sizes)
|
@@ -123,12 +135,12 @@ module Emojidex
|
|
123
135
|
@paths[:png].include?(size) && @paths[:png][size].nil? == false
|
124
136
|
@paths[:png][size] = "#{Dir.pwd}/#{size}/#{@code}.png"
|
125
137
|
end
|
126
|
-
next if File.exist?(@paths[:png][size]) &&
|
127
|
-
!@
|
128
|
-
@
|
138
|
+
next if File.exist?(@paths[:png][size]) && (
|
139
|
+
!@remote_checksums[:png][size].nil? &&
|
140
|
+
@remote_checksums[:png][size] == generate_checksum(:png, size))
|
129
141
|
response = Emojidex::Service::Transactor.download("#{size}/#{@code}.png")
|
130
142
|
File.open(@paths[:png][size], 'wb') { |fp| fp.write(response.body) }
|
131
|
-
|
143
|
+
generate_checksum(:png, size)
|
132
144
|
end
|
133
145
|
end
|
134
146
|
|
@@ -13,17 +13,23 @@ module Emojidex
|
|
13
13
|
|
14
14
|
@status = ''
|
15
15
|
|
16
|
-
_init_emoji
|
17
|
-
_init_user_info
|
18
|
-
_init_endpoint
|
19
|
-
_init_cache
|
20
|
-
|
21
16
|
@auto_cache = @opts[:auto_cache] || true
|
22
17
|
@opts.delete(:auto_cache)
|
23
18
|
|
19
|
+
@formats = @opts[:formats] || Emojidex::Defaults.selected_formats
|
20
|
+
@opts.delete(:formats)
|
21
|
+
|
22
|
+
@sizes = @opts[:sizes] || Emojidex::Defaults.selected_sizes
|
23
|
+
@opts.delete(:sizes)
|
24
|
+
|
24
25
|
auto_init = @opts[:auto_init] || true
|
25
26
|
@opts.delete(:auto_init)
|
26
27
|
|
28
|
+
_init_emoji
|
29
|
+
_init_user_info
|
30
|
+
_init_endpoint
|
31
|
+
_init_cache
|
32
|
+
|
27
33
|
more if auto_init
|
28
34
|
|
29
35
|
@emoji
|
@@ -49,6 +55,10 @@ module Emojidex
|
|
49
55
|
end
|
50
56
|
|
51
57
|
_process_moji_page(moji_page)
|
58
|
+
|
59
|
+
cache! if @auto_cache
|
60
|
+
|
61
|
+
moji_page
|
52
62
|
end
|
53
63
|
|
54
64
|
private
|
@@ -89,7 +99,6 @@ module Emojidex
|
|
89
99
|
end
|
90
100
|
|
91
101
|
add_emoji(moji_page)
|
92
|
-
cache! if @auto_cache
|
93
102
|
moji_page
|
94
103
|
end
|
95
104
|
|
@@ -1,4 +1,6 @@
|
|
1
1
|
require 'faraday'
|
2
|
+
require 'typhoeus'
|
3
|
+
require 'typhoeus/adapters/faraday'
|
2
4
|
require 'json'
|
3
5
|
require_relative '../../emojidex'
|
4
6
|
require_relative 'error'
|
@@ -8,7 +10,7 @@ module Emojidex
|
|
8
10
|
# API transaction utility
|
9
11
|
class Transactor
|
10
12
|
@@connection = nil
|
11
|
-
@@retries =
|
13
|
+
@@retries = 10
|
12
14
|
|
13
15
|
@@settings = {
|
14
16
|
api: {
|
@@ -56,10 +58,10 @@ module Emojidex
|
|
56
58
|
return @@connection if @@connection
|
57
59
|
@@connection = Faraday.new do |conn|
|
58
60
|
conn.request :url_encoded
|
59
|
-
conn.request :retry, max: @@retries, interval: 0.05, interval_randomness: 0.
|
61
|
+
conn.request :retry, max: @@retries, interval: 0.05, interval_randomness: 0.5,
|
60
62
|
backoff_factor: 2
|
61
63
|
# conn.response :logger
|
62
|
-
conn.adapter Faraday.default_adapter
|
64
|
+
conn.adapter :typhoeus #Faraday.default_adapter
|
63
65
|
end
|
64
66
|
_kludge_windows if Gem.win_platform?
|
65
67
|
@@connection
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: emojidex
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Rei Kagetsuki
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-05-
|
11
|
+
date: 2016-05-09 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: faraday
|
@@ -50,6 +50,20 @@ dependencies:
|
|
50
50
|
- - "~>"
|
51
51
|
- !ruby/object:Gem::Version
|
52
52
|
version: 0.10.0
|
53
|
+
- !ruby/object:Gem::Dependency
|
54
|
+
name: typhoeus
|
55
|
+
requirement: !ruby/object:Gem::Requirement
|
56
|
+
requirements:
|
57
|
+
- - ">="
|
58
|
+
- !ruby/object:Gem::Version
|
59
|
+
version: '0'
|
60
|
+
type: :runtime
|
61
|
+
prerelease: false
|
62
|
+
version_requirements: !ruby/object:Gem::Requirement
|
63
|
+
requirements:
|
64
|
+
- - ">="
|
65
|
+
- !ruby/object:Gem::Version
|
66
|
+
version: '0'
|
53
67
|
description: emojidex emoji handling, search and lookup, listing and caching functionality
|
54
68
|
and user info (favorites/etc).
|
55
69
|
email: info@emojidex.com
|