google_web_translate 0.2.2 → 0.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9de16ca14cfa21f2f4e2feba9fb67f1e662d37bc
-  data.tar.gz: a80a769dbb0f387f9c5c12c3baa7dc31befd0cf6
+  metadata.gz: 4fa88abae6fc3035a666db53549211dd084bbaaf
+  data.tar.gz: 4942b2dd7b155f4507d8fd1b2c902c7e66074e98
 SHA512:
-  metadata.gz: 576ad67066a1a4ba0fd953743a06a9fc713a229e6db732b5805a77b132b4984a6d7c8f4966d2f35b64f6cca9915888ee10efb3eb542c3896bfc6d3fc7d8af23c
-  data.tar.gz: 55a74de3fddd74d6e179cea599ad82ee76e19f0799de07c66f22c25c93593a06515172456a3d6dc0705af963d1c2c54921bb996aab70128b9165cc10593e95cd
+  metadata.gz: 857468dc26b8538e2be64454b1b1db9dc65ada47a53d63766c351e813ff9a5010e1759333aa0a80b194faae9adf26b895ef69aaf63c524398467016f2a04e394
+  data.tar.gz: e37080f63adfbf5e0464b2e9f8340f7b22160e25ba36b736dd6e16404fb13e92359f80f4b9e6a3485ea1c9a1b4176dd7dc55f0360949637aae210f6a552b9287

data/.gitignore

@@ -9,6 +9,7 @@
 /generated.js
 /spec.out
 /Gemfile.lock
+/data/server_data.txt
 
 # rspec failure tracking
 .rspec_status

data/Gemfile

@@ -17,5 +17,8 @@ end
 optional_gem 'therubyracer', platform: :ruby
 optional_gem 'therubyrhino', platform: :jruby
 
+# optional c extensions
+optional_gem 'concurrent-ruby-ext', platform: :ruby
+
 # Specify your gem's dependencies in google-web-translate.gemspec
 gemspec

data/data/urls.txt

@@ -0,0 +1,122 @@
+google.ae
+google.am
+google.as
+google.at
+google.at
+google.az
+google.ba
+google.be
+google.bg
+google.ca
+google.cd
+google.ch
+google.ci
+google.cl
+google.cn
+google.co.cr
+google.co.id
+google.co.il
+google.co.in
+google.co.jp
+google.co.ke
+google.co.kr
+google.com
+google.co.ma
+google.com.ai
+google.com.ar
+google.com.au
+google.com.bd
+google.com.bh
+google.com.bn
+google.com.bo
+google.com.br
+google.com.co
+google.com.cu
+google.com.do
+google.com.ec
+google.com.eg
+google.com.et
+google.com.gi
+google.com.gt
+google.com.hk
+google.com.jm
+google.com.kh
+google.com.ly
+google.com.mt
+google.com.mx
+google.com.my
+google.com.na
+google.com.ng
+google.com.ni
+google.com.om
+google.com.pa
+google.com.pe
+google.com.ph
+google.com.pk
+google.com.pr
+google.com.py
+google.com.qa
+google.com.sa
+google.com.sg
+google.com.sv
+google.com.tr
+google.com.tw
+google.com.ua
+google.com.uy
+google.com.vc
+google.com.vn
+google.co.nz
+google.co.th
+google.co.ug
+google.co.uk
+google.co.uz
+google.co.ve
+google.co.za
+google.co.zm
+google.cz
+google.de
+google.dj
+google.dk
+google.ee
+google.es
+google.fi
+google.fr
+google.ge
+google.gm
+google.gp
+google.gr
+google.hn
+google.hr
+google.ht
+google.hu
+google.ie
+google.is
+google.it
+google.je
+google.jo
+google.kz
+google.li
+google.lk
+google.lt
+google.lu
+google.lv
+google.ma
+google.md
+google.mn
+google.mu
+google.nl
+google.no
+google.nu
+google.pl
+google.pl
+google.pt
+google.ro
+google.ru
+google.rw
+google.se
+google.si
+google.sk
+google.sm
+google.sn
+google.to
+google.tt

data/google-web-translate.gemspec

@@ -1,30 +1,31 @@
-
-lib = File.expand_path('../lib', __FILE__)
-$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require 'google_web_translate/version'
-
-Gem::Specification.new do |spec|
-  spec.name          = 'google_web_translate'
-  spec.version       = GoogleWebTranslate::VERSION
-  spec.authors       = ['Andrew']
-  spec.email         = ['sobakasu@gmail.com']
-
-  spec.summary       = 'Text translation using the google web interface'
-  spec.homepage      = 'https://github.com/sobakasu/google_web_translate'
-  spec.license       = 'MIT'
-  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
-    f.match(%r{^(test|spec|features)/})
-  end
-  spec.bindir        = 'bin'
-  spec.executables   = %w[google_web_translate]
-  spec.require_paths = ['lib']
-
-  spec.add_development_dependency 'bundler', '~> 1.16'
-  spec.add_development_dependency 'rake', '~> 10.0'
-  spec.add_development_dependency 'rspec', '~> 3.0'
-  spec.add_development_dependency 'simplecov'
-  spec.add_development_dependency 'webmock'
-
-  spec.add_dependency 'execjs'
-  spec.add_dependency 'thor'
-end
+
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'google_web_translate/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'google_web_translate'
+  spec.version       = GoogleWebTranslate::VERSION
+  spec.authors       = ['Andrew']
+  spec.email         = ['sobakasu@gmail.com']
+
+  spec.summary       = 'Text translation using the google web interface'
+  spec.homepage      = 'https://github.com/sobakasu/google_web_translate'
+  spec.license       = 'MIT'
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = 'bin'
+  spec.executables   = %w[google_web_translate]
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'bundler', '~> 1.16'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec', '~> 3.0'
+  spec.add_development_dependency 'simplecov'
+  spec.add_development_dependency 'webmock'
+
+  spec.add_dependency 'concurrent-ruby'
+  spec.add_dependency 'execjs'
+  spec.add_dependency 'thor'
+end

data/lib/google_web_translate.rb

@@ -1,5 +1,6 @@
-require 'google_web_translate/version.rb'
-require 'google_web_translate/string_escaping.rb'
-require 'google_web_translate/http_client.rb'
-require 'google_web_translate/result.rb'
-require 'google_web_translate/api.rb'
+require 'google_web_translate/version.rb'
+require 'google_web_translate/server_list.rb'
+require 'google_web_translate/string_escaping.rb'
+require 'google_web_translate/http_client.rb'
+require 'google_web_translate/result.rb'
+require 'google_web_translate/api.rb'

data/lib/google_web_translate/api.rb

@@ -1,138 +1,142 @@
-require 'execjs'
-require 'json'
-
-module GoogleWebTranslate
-  # interface to the google web translation api
-  class API
-    def initialize(options = {})
-      @dt = options[:dt] || DEFAULT_DT
-      @token_ttl = options[:token_ttl] || DEFAULT_TOKEN_TTL
-      @debug = options[:debug]
-      @http_client = options[:http_client] || HTTPClient.new(options)
-    end
-
-    def translate(string, from, to)
-      data = fetch_translation(string, from, to)
-      Result.new(data)
-    end
-
-    def languages
-      @languages ||= begin
-        html = fetch_main
-        html.scan(/\['(\w{2})','(\w{2})'\]/).flatten.uniq.sort
-      end
-    end
-
-    private
-
-    URL_MAIN = 'https://translate.google.com'.freeze
-    URL_TRANSLATE_1 = URL_MAIN + '/translate_a/single'.freeze
-    DEFAULT_DT = %w[at bd ex ld md qca rw rm ss t].freeze
-    DEFAULT_TOKEN_TTL = 3600
-
-    def fetch_translation(string, from, to)
-      json = fetch_url_body(translate_url(string, from, to))
-      # File.open("response.json", "w") { |f| f.puts json }
-      JSON.parse(json)
-    end
-
-    def fetch_url_response(url)
-      @http_client.get(url.to_s)
-    end
-
-    def fetch_url_body(url)
-      uri = URI.parse(url)
-      uri = URI.join(URL_MAIN, url) if uri.relative?
-      debug("fetch #{uri}")
-      response = fetch_url_response(uri)
-      # debug("response: #{response.body}")
-      response.body
-    end
-
-    def valid_token?
-      @token_updated_at && Time.now - @token_updated_at < @token_ttl
-    end
-
-    def fetch_main(options = {})
-      @html = nil if options[:no_cache]
-      @html ||= fetch_url_body(URL_MAIN)
-    end
-
-    def fetch_desktop_module(html)
-      html =~ /([^="]*desktop_module_main.js)/
-      url = Regexp.last_match(1)
-      raise 'unable to find desktop module' unless url
-      fetch_url_body(url)
-    end
-
-    def munge_module(js)
-      js.gsub(/((?:var\s+)?\w+\s*=\s*\w+\.createElement.*?;)/) do |_i|
-        'return "";'
-      end
-    end
-
-    def compile_js(html)
-      desktop_module_js = munge_module(fetch_desktop_module(html))
-      window_js = File.read(File.join(__dir__, '..', 'js', 'window.js'))
-      js = window_js + desktop_module_js
-      File.open('generated.js', 'w') { |f| f.puts(js) } if debug?
-      ExecJS.compile(js)
-    end
-
-    def update_token
-      # download main page
-      html = fetch_main(no_cache: true)
-      # extract tkk from html
-      @tkk = extract_tkk(html)
-      # compile desktop module javascript
-      @js_context = compile_js(html)
-      @token_updated_at = Time.now
-    end
-
-    def tk(string)
-      update_token unless valid_token?
-      @js_context.call('setWindowProperty', 'TKK', @tkk)
-      # tk = @js_context.call("wq", string)
-      tk = @js_context.call('generateToken', string, @tkk)
-      (tk.split('=') || [])[1]
-    end
-
-    def tk_js
-      File.read(File.join(__dir__, 'google_web.js'))
-    end
-
-    def extract_tkk(html)
-      raise 'TKK not found' unless html =~ /TKK=eval\('(.*?)'\);/
-      tkk_code = Regexp.last_match(1)
-      # tkk_code = Translatomatic::StringEscaping.unescape(tkk_code)
-      tkk_code = StringEscaping.unescape(tkk_code)
-      debug("tkk code unescaped: #{tkk_code}")
-      tkk = ExecJS.eval(tkk_code)
-      # tkk = context.call(nil)
-      debug("evaluated tkk: #{tkk}")
-      tkk
-    end
-
-    def translate_url(string, from, to)
-      tk = tk(string)
-      debug("tk: #{tk}")
-      query = {
-        sl: from, tl: to, ie: 'UTF-8', oe: 'UTF-8',
-        q: string, dt: @dt, tk: tk,
-        # not sure what these are for
-        client: 't', hl: 'en', otf: 1, ssel: 4, tsel: 6, kc: 5
-      }
-      url = URI.parse(URL_TRANSLATE_1)
-      url.query = URI.encode_www_form(query)
-      url.to_s
-    end
-
-    def debug(msg)
-      puts msg if debug?
-    end
-
-    def debug?
-      @debug
-    end
-  end
-end
+require 'execjs'
+require 'json'
+
+module GoogleWebTranslate
+  # interface to the google web translation api
+  class API
+    def initialize(options = {})
+      @dt = options[:dt] || DEFAULT_DT
+      @token_ttl = options[:token_ttl] || DEFAULT_TOKEN_TTL
+      @debug = options[:debug]
+      @http_client = options[:http_client] || HTTPClient.new(options)
+      @rate_limit = options[:rate_limit] || DEFAULT_RATE_LIMIT
+    end
+
+    def translate(string, from, to)
+      data = fetch_translation(string, from, to)
+      Result.new(data)
+    end
+
+    def languages
+      @languages ||= begin
+        html = fetch_main
+        html.scan(/\['(\w{2})','(\w{2})'\]/).flatten.uniq.sort
+      end
+    end
+
+    private
+
+    URL_MAIN = 'https://translate.google.com'.freeze
+    TRANSLATE_PATH = '/translate_a/single'.freeze
+    DEFAULT_DT = %w[at bd ex ld md qca rw rm ss t].freeze
+    DEFAULT_TOKEN_TTL = 3600
+    DEFAULT_RATE_LIMIT = 5
+
+    def fetch_translation(string, from, to)
+      server = ServerList.next_server(@rate_limit)
+      json = fetch_url_body(translate_url(server, string, from, to))
+      # File.write("response.json", json) if debug?
+      debug("response: #{json}")
+      JSON.parse(json)
+    end
+
+    def fetch_url_response(url)
+      @http_client.get(url.to_s)
+    end
+
+    def fetch_url_body(url)
+      uri = URI.parse(url)
+      uri = URI.join(URL_MAIN, url) if uri.relative?
+      debug("fetch #{uri}")
+      response = fetch_url_response(uri)
+      response.body
+    end
+
+    def valid_token?
+      @token_updated_at && Time.now - @token_updated_at < @token_ttl
+    end
+
+    def fetch_main(options = {})
+      @html = nil if options[:no_cache]
+      @html ||= fetch_url_body(URL_MAIN)
+    end
+
+    def fetch_desktop_module(html)
+      html =~ /([^="]*desktop_module_main.js)/
+      url = Regexp.last_match(1)
+      raise 'unable to find desktop module' unless url
+      fetch_url_body(url)
+    end
+
+    def munge_module(js)
+      js.gsub(/((?:var\s+)?\w+\s*=\s*\w+\.createElement.*?;)/) do |_i|
+        'return "";'
+      end
+    end
+
+    def compile_js(html)
+      desktop_module_js = munge_module(fetch_desktop_module(html))
+      window_js = File.read(File.join(__dir__, '..', 'js', 'window.js'))
+      js = window_js + desktop_module_js
+      # File.write('generated.js', js) if debug?
+      ExecJS.compile(js)
+    end
+
+    def update_token
+      # download main page
+      html = fetch_main(no_cache: true)
+      # extract tkk from html
+      @tkk = extract_tkk(html)
+      # compile desktop module javascript
+      @js_context = compile_js(html)
+      @token_updated_at = Time.now
+    end
+
+    def tk(string)
+      update_token unless valid_token?
+      @js_context.call('setWindowProperty', 'TKK', @tkk)
+      # tk = @js_context.call("wq", string)
+      tk = @js_context.call('generateToken', string, @tkk)
+      (tk.split('=') || [])[1]
+    end
+
+    def tk_js
+      File.read(File.join(__dir__, 'google_web.js'))
+    end
+
+    def extract_tkk(html)
+      raise 'TKK not found' unless html =~ /TKK=eval\('(.*?)'\);/
+      tkk_code = Regexp.last_match(1)
+      # tkk_code = Translatomatic::StringEscaping.unescape(tkk_code)
+      tkk_code = StringEscaping.unescape(tkk_code)
+      debug("tkk code unescaped: #{tkk_code}")
+      tkk = ExecJS.eval(tkk_code)
+      # tkk = context.call(nil)
+      debug("evaluated tkk: #{tkk}")
+      tkk
+    end
+
+    def translate_url(server, string, from, to)
+      tk = tk(string)
+      debug("tk: #{tk}")
+      query = {
+        sl: from, tl: to, ie: 'UTF-8', oe: 'UTF-8',
+        q: string, dt: @dt, tk: tk,
+        # not sure what these are for
+        client: 't', hl: 'en', otf: 1, ssel: 4, tsel: 6, kc: 5
+      }
+      url = "https://#{server.host}" + TRANSLATE_PATH
+      uri = URI.parse(url)
+      uri.query = URI.encode_www_form(query)
+      uri.to_s
+    end
+
+    def debug(msg)
+      puts msg if debug?
+    end
+
+    def debug?
+      @debug
+    end
+  end
+end

data/lib/google_web_translate/cli.rb

@@ -1,18 +1,18 @@
-require 'thor'
-require 'pp'
-
-module GoogleWebTranslate
-  # Command line interface
-  class CLI < Thor
-    desc 'string from to', 'translate a string from one language to another'
-    method_option :dt, type: :array, desc: 'data types'
-    def translate(string, from, to)
-      api_options = { debug: ENV['DEBUG'] }
-      api_options[:dt] = options[:dt] if options[:dt]
-
-      api = API.new(api_options)
-      result = api.translate(string, from, to)
-      pp result.to_h
-    end
-  end
-end
+require 'thor'
+require 'pp'
+
+module GoogleWebTranslate
+  # Command line interface
+  class CLI < Thor
+    desc 'string from to', 'translate a string from one language to another'
+    method_option :dt, type: :array, desc: 'data types'
+    def translate(string, from, to)
+      api_options = { debug: ENV['DEBUG'] }
+      api_options[:dt] = options[:dt] if options[:dt]
+
+      api = API.new(api_options)
+      result = api.translate(string, from, to)
+      pp result.to_h
+    end
+  end
+end

data/lib/google_web_translate/http_client.rb

@@ -1,26 +1,26 @@
-require 'net/http'
-
-module GoogleWebTranslate
-  # HTTP client functionality
-  class HTTPClient
-    def self.user_agent
-      gem_version = "GoogleWebTranslate/#{VERSION}"
-      platform_version = "(#{RUBY_PLATFORM}) #{RUBY_ENGINE}/#{RUBY_VERSION}"
-      gem_version + ' ' + platform_version
-    end
-
-    def initialize(options = {})
-      @user_agent = options[:user_agent] || self.class.user_agent
-    end
-
-    def get(url)
-      uri = URI.parse(url)
-      request = Net::HTTP::Get.new(uri)
-      request['User-Agent'] = @user_agent
-      options = { use_ssl: uri.scheme == 'https' }
-      Net::HTTP.start(uri.host, uri.port, options) do |http|
-        http.request(request)
-      end
-    end
-  end
-end
+require 'net/http'
+
+module GoogleWebTranslate
+  # HTTP client functionality
+  class HTTPClient
+    def self.user_agent
+      gem_version = "GoogleWebTranslate/#{VERSION}"
+      platform_version = "(#{RUBY_PLATFORM}) #{RUBY_ENGINE}/#{RUBY_VERSION}"
+      gem_version + ' ' + platform_version
+    end
+
+    def initialize(options = {})
+      @user_agent = options[:user_agent] || self.class.user_agent
+    end
+
+    def get(url)
+      uri = URI.parse(url)
+      request = Net::HTTP::Get.new(uri)
+      request['User-Agent'] = @user_agent
+      options = { use_ssl: uri.scheme == 'https' }
+      Net::HTTP.start(uri.host, uri.port, options) do |http|
+        http.request(request)
+      end
+    end
+  end
+end

data/lib/google_web_translate/result.rb

@@ -1,54 +1,54 @@
-module GoogleWebTranslate
-  # Translation results
-  class Result
-    attr_reader :raw
-
-    # @private
-    DATA_INDICES = {
-      translation: [0, 0, 0], # dt:t
-      alternatives: [5, 0, 2], # dt:at
-      dictionary: [1], # dt: bd
-      synonyms: [11], # dt:ss
-      definitions: [12, 0], # dt:md
-      examples: [13, 0], # dt:ex
-      see_also: [14, 0], # dt:rw
-    }.freeze
-
-    DATA_INDICES.each_key { |key| attr_reader key }
-
-    def initialize(data)
-      @raw = data
-      @keys = []
-      @properties = {}
-
-      DATA_INDICES.each do |key, indices|
-        indices = indices.dup
-        extract_data(key, *indices)
-      end
-
-      @alternatives = @alternatives.collect { |i| i[0] } if @alternatives
-      @keys.each { |key| @properties[key] = instance_variable_get("@#{key}") }
-    end
-
-    def to_h
-      @properties
-    end
-
-    private
-
-    def extract_data(name, *indices)
-      value = array_value(@raw, *indices)
-      return if value.nil?
-      instance_variable_set("@#{name}", value)
-      @keys.push(name)
-    end
-
-    def array_value(array, *indices)
-      return nil if array.nil?
-      index = indices.shift
-      value = array[index]
-      return value if indices.empty?
-      array_value(value, *indices)
-    end
-  end
-end
+module GoogleWebTranslate
+  # Translation results
+  class Result
+    attr_reader :raw
+
+    # @private
+    DATA_INDICES = {
+      translation: [0, 0, 0], # dt:t
+      alternatives: [5, 0, 2], # dt:at
+      dictionary: [1], # dt: bd
+      synonyms: [11], # dt:ss
+      definitions: [12, 0], # dt:md
+      examples: [13, 0], # dt:ex
+      see_also: [14, 0], # dt:rw
+    }.freeze
+
+    DATA_INDICES.each_key { |key| attr_reader key }
+
+    def initialize(data)
+      @raw = data
+      @keys = []
+      @properties = {}
+
+      DATA_INDICES.each do |key, indices|
+        indices = indices.dup
+        extract_data(key, *indices)
+      end
+
+      @alternatives = @alternatives.collect { |i| i[0] } if @alternatives
+      @keys.each { |key| @properties[key] = instance_variable_get("@#{key}") }
+    end
+
+    def to_h
+      @properties
+    end
+
+    private
+
+    def extract_data(name, *indices)
+      value = array_value(@raw, *indices)
+      return if value.nil?
+      instance_variable_set("@#{name}", value)
+      @keys.push(name)
+    end
+
+    def array_value(array, *indices)
+      return nil if array.nil?
+      index = indices.shift
+      value = array[index]
+      return value if indices.empty?
+      array_value(value, *indices)
+    end
+  end
+end

data/lib/google_web_translate/server_list.rb

@@ -0,0 +1,146 @@
+require 'concurrent'
+require 'resolv'
+require 'json'
+
+module GoogleWebTranslate
+  # @private
+  SERVER_ATTRIBUTES = %i[host ip resolved_at last_used_at
+                         counter available].freeze
+
+  Server = Struct.new(*SERVER_ATTRIBUTES) do
+    def to_json(*args)
+      result = {}
+      each_pair { |key, value| result[key] = value }
+      result.to_json(args)
+    end
+  end
+
+  class ServerList
+    class << self
+      def servers
+        update_servers if @servers.nil?
+        @servers.dup
+      end
+
+      def next_server(rate_limit = nil)
+        @mutex ||= Mutex.new
+        @mutex.synchronize do
+          @counter ||= 0
+          @counter += 1
+
+          list = servers.sort_by { |i| i.counter || 0 }
+          server = list[0]
+          server.counter = @counter
+          sleep(rate_limit_delay(server, rate_limit))
+          server.last_used_at = Time.now
+          server
+        end
+      end
+
+      private
+
+      MAX_TTL = 86_400
+
+      def rate_limit_delay(server, rate_limit)
+        return 0 unless rate_limit && server.last_used_at
+        delay = rate_limit - (Time.now - server.last_used_at)
+        delay < 0 || ENV['TEST'] ? 0 : delay
+      end
+
+      def update_servers
+        server_list = read_server_data
+        pool = Concurrent::CachedThreadPool.new
+        # puts "updating #{server_list.length} servers"
+        server_list.each do |server|
+          pool.post { update_server(server) }
+        end
+
+        pool.shutdown
+        pool.wait_for_termination
+        @servers = unique_servers(server_list)
+        # puts "#{@servers.length} unique servers found"
+        save_server_data(server_list)
+      end
+
+      def update_server(server)
+        now = Time.now.to_i
+        if server.resolved_at.nil? ||
+           now - server.resolved_at > MAX_TTL || !server.available
+          server.resolved_at = now
+          server.ip = resolve_ip(server.host)
+        end
+        server.available = true
+      rescue Resolv::ResolvError
+        # puts "server #{server.host} is unavailable: #{e}"
+        server.available = false
+      end
+
+      def data_dir
+        File.join(__dir__, '..', '..', 'data')
+      end
+
+      def server_data_path
+        File.join(data_dir, 'server_data.txt')
+      end
+
+      def url_list_path
+        File.join(data_dir, 'urls.txt')
+      end
+
+      def hostnames
+        names = []
+        lines = File.read(url_list_path).split(/[\r\n]+/)
+        lines.each do |host|
+          next unless host && !host.empty?
+          names << "translate.#{host}"
+        end
+        names
+      end
+
+      def unique_servers(list)
+        server_by_ip = {}
+        list.each do |server|
+          next unless server.available
+          server_by_ip[server.ip] = server
+        end
+        server_by_ip.values
+      end
+
+      def initial_data
+        hostnames.collect do |host|
+          server = Server.new
+          server.host = host
+          server
+        end
+      end
+
+      def read_server_data
+        return initial_data unless File.exist?(server_data_path)
+        data = JSON.parse(File.read(server_data_path))
+        server_list = []
+        data.each do |entry|
+          attributes = SERVER_ATTRIBUTES.collect { |i| entry[i.to_s] }
+          server = Server.new(*attributes)
+          next unless server.host && !server.host.empty?
+          server.counter = 0
+          server_list << server
+        end
+        server_list
+      end
+
+      def save_server_data(servers)
+        File.write(server_data_path, servers.to_json)
+      end
+
+      def resolver
+        resolver = Resolv::DNS.new
+        resolver.timeouts = 5
+        resolver
+      end
+
+      def resolve_ip(host)
+        resolver.getaddress(host).to_s
+      end
+    end
+  end
+end

data/lib/google_web_translate/version.rb

@@ -1,3 +1,3 @@
-module GoogleWebTranslate
-  VERSION = '0.2.2'.freeze
-end
+module GoogleWebTranslate
+  VERSION = '0.2.3'.freeze
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: google_web_translate
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.2.3
 platform: ruby
 authors:
 - Andrew
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-01-23 00:00:00.000000000 Z
+date: 2018-02-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -80,6 +80,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: execjs
   requirement: !ruby/object:Gem::Requirement
@@ -128,12 +142,14 @@ files:
 - bin/console
 - bin/google_web_translate
 - bin/setup
+- data/urls.txt
 - google-web-translate.gemspec
 - lib/google_web_translate.rb
 - lib/google_web_translate/api.rb
 - lib/google_web_translate/cli.rb
 - lib/google_web_translate/http_client.rb
 - lib/google_web_translate/result.rb
+- lib/google_web_translate/server_list.rb
 - lib/google_web_translate/string_escaping.rb
 - lib/google_web_translate/version.rb
 - lib/js/window.js