fontist 1.10.0 → 1.11.3

data/exe/fontist

@@ -3,5 +3,27 @@
 require "fontist"
 require "fontist/cli"
 
-status_code = Fontist::CLI.start(ARGV)
-exit status_code.is_a?(Integer) ? status_code : 1
+fontist_cli = proc {
+  status_code = Fontist::CLI.start(ARGV)
+  exit status_code.is_a?(Integer) ? status_code : 1
+}
+
+if ENV["SOCKS_PROXY"]
+  require "socksify"
+  require "uri"
+  begin
+    proxy = URI.parse(ENV["SOCKS_PROXY"])
+    if proxy.userinfo
+      user, pass = proxy.userinfo.split(":")
+      TCPSocket::socks_username = user
+      TCPSocket::socks_password = pass
+    end
+    Socksify::proxy(proxy.host, proxy.port, &fontist_cli)
+  rescue URI::InvalidURIError
+    warn "Value of ENV.SOCKS_PROXY=#{ENV['SOCKS_PROXY']} is invalid! Droping it"
+    ENV.delete("SOCKS_PROXY")
+    fontist_cli.call
+  end
+else
+  fontist_cli.call
+end

data/fontist.gemspec

@@ -28,20 +28,22 @@ Gem::Specification.new do |spec|
   spec.test_files    = `git ls-files -- {spec}/*`.split("\n")
 
   spec.add_runtime_dependency "down", "~> 5.0"
+  spec.add_runtime_dependency "extract_ttc", "~> 0.1"
   spec.add_runtime_dependency "thor", "~> 1.0.1"
   spec.add_runtime_dependency "git", "~> 1.0"
   spec.add_runtime_dependency "ttfunk", "~> 1.6"
   spec.add_runtime_dependency "excavate", "~> 0.1"
 
-  spec.add_development_dependency "extract_ttc", "~> 0.1"
   spec.add_development_dependency "pry"
   spec.add_development_dependency "bundler", "~> 2.0"
   spec.add_development_dependency "gem-release"
   spec.add_development_dependency "nokogiri", "~> 1.0"
   spec.add_development_dependency "rake", "~> 13"
   spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "rspec-benchmark", "~> 0.6"
   spec.add_development_dependency "rubocop", "1.5.2"
   spec.add_development_dependency "rubocop-rails"
   spec.add_development_dependency "rubocop-performance"
-  spec.add_development_dependency "ruby-protocol-buffers", "~> 1.0"
+
+  spec.add_runtime_dependency "socksify"
 end

data/lib/fontist/cli.rb

@@ -1,5 +1,6 @@
 require "thor"
 require "fontist/repo_cli"
+require "fontist/google_cli"
 
 module Fontist
   class CLI < Thor
@@ -149,18 +150,9 @@ module Fontist
       It is done automatically when formulas are updated, or private formulas
       are set up.
     LONGDESC
-    option :main_repo, type: :boolean,
-                       desc: "Updates indexes in the main repo (for backward " \
-                             "compatibility with versions prior to 1.9)"
     def rebuild_index
       handle_class_options(options)
-
-      if options[:main_repo]
-        Fontist::Index.rebuild_for_main_repo
-      else
-        Fontist::Index.rebuild
-      end
-
+      Fontist::Index.rebuild
       Fontist.ui.say("Formula index has been rebuilt.")
       STATUS_SUCCESS
     end
@@ -175,6 +167,9 @@ module Fontist
     desc "repo SUBCOMMAND ...ARGS", "Manage custom repositories"
     subcommand "repo", Fontist::RepoCLI
 
+    desc "google SUBCOMMAND ...ARGS", "Manage Google formulas"
+    subcommand "google", Fontist::GoogleCLI
+
     private
 
     def handle_class_options(options)

data/lib/fontist/formula.rb

@@ -103,6 +103,10 @@ module Fontist
       @fonts ||= Helpers.parse_to_object(hash_collection_fonts + hash_fonts)
     end
 
+    def digest
+      @data["digest"]
+    end
+
     private
 
     def default_key

data/lib/fontist/google_cli.rb

@@ -0,0 +1,29 @@
+module Fontist
+  class GoogleCLI < Thor
+    class_option :formulas_path, type: :string, desc: "Path to formulas"
+
+    desc "check", "Check Google fonts for updates"
+    def check
+      handle_class_options(options)
+      require "fontist/import/google_check"
+      Fontist::Import::GoogleCheck.new.call
+      CLI::STATUS_SUCCESS
+    end
+
+    desc "import", "Import Google fonts"
+    def import
+      handle_class_options(options)
+      require "fontist/import/google_import"
+      Fontist::Import::GoogleImport.new.call
+      CLI::STATUS_SUCCESS
+    end
+
+    private
+
+    def handle_class_options(options)
+      if options[:formulas_path]
+        Fontist.formulas_path = Pathname.new(options[:formulas_path])
+      end
+    end
+  end
+end

data/lib/fontist/import/files/collection_file.rb

@@ -11,7 +11,7 @@ module Fontist
         def initialize(path)
           @path = path
           @fonts = read
-          @extension = "ttc"
+          @extension = detect_extension
         end
 
         def filename
@@ -46,6 +46,15 @@ module Fontist
             File.join(tmp_dir, filename)
           end
         end
+
+        def detect_extension
+          base_extension = "ttc"
+
+          file_extension = File.extname(File.basename(@path)).sub(/^\./, "")
+          return file_extension if file_extension.casecmp?(base_extension)
+
+          base_extension
+        end
       end
     end
   end

data/lib/fontist/import/formula_builder.rb

@@ -1,3 +1,4 @@
+require "shellwords"
 require_relative "text_helper"
 
 module Fontist
@@ -5,7 +6,7 @@ module Fontist
     class FormulaBuilder
       FORMULA_ATTRIBUTES = %i[name description homepage resources
                               font_collections fonts extract copyright
-                              license_url open_license].freeze
+                              license_url open_license digest command].freeze
 
       attr_accessor :archive,
                     :url,
@@ -49,7 +50,7 @@ module Fontist
       end
 
       def homepage
-        both_fonts.first.homepage
+        both_fonts.map(&:homepage).compact.first
       end
 
       def resources
@@ -59,6 +60,18 @@ module Fontist
       end
 
       def resource_options
+        if @options[:skip_sha]
+          resource_options_without_sha
+        else
+          resource_options_with_sha
+        end
+      end
+
+      def resource_options_without_sha
+        { urls: [@url] + mirrors }
+      end
+
+      def resource_options_with_sha
         urls = []
         sha = []
         downloads do |url, path|
@@ -146,11 +159,11 @@ module Fontist
       end
 
       def copyright
-        both_fonts.first.copyright
+        both_fonts.map(&:copyright).compact.first
       end
 
       def license_url
-        both_fonts.first.license_url
+        both_fonts.map(&:license_url).compact.first
       end
 
       def open_license
@@ -165,6 +178,14 @@ module Fontist
 
         TextHelper.cleanup(@license_text)
       end
+
+      def digest
+        @options[:digest]
+      end
+
+      def command
+        Shellwords.shelljoin(ARGV)
+      end
     end
   end
 end

data/lib/fontist/import/google/new_fonts_fetcher.rb

@@ -1,4 +1,3 @@
-require_relative "fonts_public.pb"
 require_relative "../google"
 require_relative "../otf_parser"
 
@@ -6,7 +5,7 @@ module Fontist
   module Import
     module Google
       class NewFontsFetcher
-        REPO_PATH = Fontist.root_path.join("tmp", "fonts")
+        REPO_PATH = Fontist.fontist_path.join("google", "fonts")
         REPO_URL = "https://github.com/google/fonts.git".freeze
         SKIPLIST_PATH = File.expand_path("skiplist.yml", __dir__)
 
@@ -25,7 +24,8 @@ module Fontist
           if Dir.exist?(REPO_PATH)
             `cd #{REPO_PATH} && git pull`
           else
-            `git clone #{REPO_URL} #{REPO_PATH}`
+            FileUtils.mkdir_p(File.dirname(REPO_PATH))
+            `git clone --depth 1 #{REPO_URL} #{REPO_PATH}`
           end
         end
 
@@ -53,23 +53,11 @@ module Fontist
         end
 
         def new?(path)
-          metadata = fetch_metadata(path)
-          return unless metadata
-
-          font_new?(metadata, path)
-        end
-
-        def fetch_metadata(path)
-          metadata_path = File.join(path, "METADATA.pb")
-          return unless File.exists?(metadata_path)
-
-          ::Google::Fonts::FamilyProto.parse_from_text(File.read(metadata_path))
-        end
-
-        def font_new?(metadata, path)
-          return if in_skiplist?(metadata.name)
-          return if up_to_date?(metadata, path)
-          return unless downloadable?(metadata.name)
+          metadata_name = Google.metadata_name(path)
+          return unless metadata_name
+          return if in_skiplist?(metadata_name)
+          return if up_to_date?(metadata_name, path)
+          return unless downloadable?(metadata_name)
 
           true
         end
@@ -79,29 +67,47 @@ module Fontist
           @skiplist.include?(name)
         end
 
-        def up_to_date?(metadata, path)
-          formula = formula(metadata.name)
+        def up_to_date?(metadata_name, path)
+          formula = formula(metadata_name)
           return false unless formula
 
-          styles = formula.fonts.map(&:styles).flatten
+          repo_digest_up_to_date?(formula, path) ||
+            fonts_up_to_date?(formula, path)
+        end
+
+        def repo_digest_up_to_date?(formula, path)
+          return unless formula.digest
 
-          styles.all? do |style|
-            style.version == otfinfo_version(font_path(style.font, path))
+          formula.digest == Google.digest(path)
+        end
+
+        def fonts_up_to_date?(formula, path)
+          styles = formula_styles(formula)
+          repo_fonts(path).all? do |font|
+            style = styles.find { |s| s.font == repo_to_archive_name(font) }
+            return false unless style
+
+            otfinfo_version(font) == style.version
           end
         end
 
-        def formula(font_name)
-          klass = font_name.gsub(/ /, "").sub(/\S/, &:upcase)
-          @formulas ||= Fontist::Formula.all
-          @formulas["Fontist::Formulas::#{klass}Font"]
+        def formula_styles(formula)
+          formula.fonts.map(&:styles).flatten
         end
 
-        def font_path(filename, directory)
-          File.join(directory, fix_variable_filename(filename))
+        def repo_fonts(path)
+          Dir.glob(File.join(path, "*.{ttf,otf}"))
         end
 
-        def fix_variable_filename(filename)
-          filename.sub("-VariableFont_wght", "[wght]")
+        def repo_to_archive_name(font_path)
+          File.basename(font_path)
+            .sub("[wght]", "-VariableFont_wght")
+            .sub("[opsz]", "-Regular-VariableFont_opsz")
+        end
+
+        def formula(font_name)
+          path = Fontist::Import::Google.formula_path(font_name)
+          Formula.new_from_file(path) if File.exist?(path)
         end
 
         def otfinfo_version(path)
@@ -110,10 +116,15 @@ module Fontist
         end
 
         def downloadable?(name)
+          retries ||= 0
+          retries += 1
           Down.open("https://fonts.google.com/download?family=#{name}")
           true
         rescue Down::NotFound
           false
+        rescue Down::TimeoutError
+          retry unless retries >= 3
+          false
         end
       end
     end

data/lib/fontist/import/google.rb

@@ -1,11 +1,29 @@
 module Fontist
   module Import
     module Google
+      def self.metadata_name(path)
+        metadata_path = File.join(path, "METADATA.pb")
+        return unless File.exists?(metadata_path)
+
+        File.foreach(metadata_path) do |line|
+          name = line.match(/^name: "(.+)"/)
+          return name[1] if name
+        end
+      end
+
       def self.formula_path(name)
         filename = name.downcase.gsub(" ", "_") + ".yml"
         Fontist.formulas_path.join("google", filename)
       end
 
+      def self.digest(path)
+        checksums = Dir.glob(File.join(path, "*.{ttf,otf,ttc}"))
+          .sort
+          .map { |x| Digest::SHA256.file(x).to_s }
+
+        Digest::SHA256.hexdigest(checksums.to_s)
+      end
+
       def self.style_version(text)
         return unless text
 

data/lib/fontist/import/google_check.rb

@@ -4,17 +4,12 @@ module Fontist
   module Import
     class GoogleCheck
       def call
-        fetch_formulas
         fonts = new_fonts
         indicate(fonts)
       end
 
       private
 
-      def fetch_formulas
-        Formula.update_formulas_repo
-      end
-
       def new_fonts
         Fontist::Import::Google::NewFontsFetcher.new(logging: true).call
       end
@@ -26,8 +21,6 @@ module Fontist
         new_paths.each do |path|
           puts path
         end
-
-        abort
       end
     end
   end

data/lib/fontist/import/google_import.rb

@@ -1,11 +1,7 @@
 require "erb"
 require_relative "google"
 require_relative "google/new_fonts_fetcher"
-require_relative "google/fonts_public.pb"
-require_relative "template_helper"
-require_relative "text_helper"
-require_relative "otf_parser"
-require_relative "otf_style"
+require_relative "create_formula"
 
 module Fontist
   module Import
@@ -13,6 +9,7 @@ module Fontist
       def call
         fonts = new_fonts
         create_formulas(fonts)
+        rebuild_index
       end
 
       private
@@ -22,161 +19,48 @@ module Fontist
       end
 
       def create_formulas(fonts)
+        return puts("Nothing to update") if fonts.empty?
+
         puts "Creating formulas..."
         fonts.each do |path|
           create_formula(path)
         end
       end
 
-      def create_formula(path)
-        puts path
-        metadata = fetch_metadata(path)
-        font = build_font(metadata, path)
-        save_formula(font)
-      end
+      def create_formula(font_path)
+        puts font_path
 
-      def fetch_metadata(path)
-        protobuf = File.read(File.join(path, "METADATA.pb"))
-        ::Google::Fonts::FamilyProto.parse_from_text(protobuf)
-      end
+        path = Fontist::Import::CreateFormula.new(
+          url(font_path),
+          name: Google.metadata_name(font_path),
+          formula_dir: formula_dir,
+          skip_sha: variable_style?(font_path),
+          digest: Google.digest(font_path),
+        ).call
 
-      def build_font(metadata, path)
-        h = from_metadata(metadata)
-          .merge(from_otfinfo(path))
-          .merge(styles: styles_from_otfinfo(path, metadata.fonts))
-          .merge(from_license(path))
-
-        OpenStruct.new(h)
+        Fontist.ui.success("Formula has been successfully created: #{path}")
       end
 
-      def from_metadata(metadata)
-        copyright = metadata.fonts.first.copyright
-
-        Hash.new.tap do |h|
-          h[:fullname] = metadata.name
-          h[:cleanname] = metadata.name.gsub(/ /, "")
-          h[:sha256] = sha256(metadata.name) unless variable_style?(metadata)
-          h[:copyright] = Fontist::Import::TextHelper.cleanup(copyright)
-        end
+      def url(path)
+        name = Google.metadata_name(path)
+        "https://fonts.google.com/download?family=#{ERB::Util.url_encode(name)}"
       end
 
-      def variable_style?(metadata)
-        metadata.fonts.any? do |s|
-          s.filename.match?(/\[(.+,)?wght\]/)
+      def formula_dir
+        @formula_dir ||= Fontist.formulas_path.join("google").tap do |path|
+          FileUtils.mkdir_p(path) unless File.exist?(path)
         end
       end
 
-      def sha256(name)
-        file = Down.download("https://fonts.google.com/download?family=#{name}",
-                             open_timeout: 10,
-                             read_timeout: 10)
-
-        Digest::SHA256.file(file).to_s
-      end
-
-      def from_license(path)
-        file = Dir.glob(File.join(path, "{OFL.txt,UFL.txt,LICENSE.txt}")).first
-        print "warn, no license, " unless file
-        return { license: "" } unless file
-
-        { license: cleanup_text(File.read(file)) }
-      end
-
-      def cleanup_text(text)
-        text.rstrip
-          .gsub("\r\n", "\n")
-          .lines
-          .map(&:rstrip)
-          .drop_while(&:empty?)
-          .join("\n")
-      end
-
-      def from_otfinfo(path)
-        font_file = Dir.glob(File.join(path, "*.ttf")).first
-        otf = OtfParser.new(font_file).call
-
-        { homepage: otf["Vendor URL"],
-          license_url: otf["License URL"] }
-      end
-
-      def styles_from_otfinfo(path, fonts)
-        fonts.map do |f|
-          file_path = File.join(path, f.filename)
-          info = OtfParser.new(file_path).call
-          OtfStyle.new(info, file_path).call
+      def variable_style?(path)
+        fonts = Dir.glob(File.join(path, "*.{ttf,otf}"))
+        fonts.any? do |font|
+          File.basename(font).match?(/\[(.+,)?(wght|opsz)\]/)
         end
       end
 
-      def save_formula(font)
-        hash = formula_hash(font)
-        path = formula_path(font.fullname)
-        save_to_path(hash, path)
-      end
-
-      def formula_hash(font)
-        stringify_keys(name: font.cleanname.sub(/\S/, &:upcase),
-                       description: font.fullname,
-                       homepage: font.homepage,
-                       resources: formula_resource(font),
-                       fonts: [yaml_font(font)],
-                       extract: { format: :zip },
-                       copyright: font.copyright,
-                       license_url: font.license_url,
-                       open_license: font.license)
-      end
-
-      def stringify_keys(hash)
-        JSON.parse(hash.to_json)
-      end
-
-      def formula_resource(font)
-        encoded_name = ERB::Util.url_encode(font.fullname)
-        url = "https://fonts.google.com/download?family=#{encoded_name}"
-
-        options = {}
-        options[:urls] = [url]
-        options[:sha256] = font.sha256 if font.sha256
-
-        { "#{font.cleanname}.zip" => options }
-      end
-
-      def yaml_font(font)
-        { name: font.fullname,
-          styles: yaml_styles(font.styles) }
-      end
-
-      def yaml_styles(styles)
-        styles.map do |s|
-          yaml_style(s)
-        end
-      end
-
-      def yaml_style(style)
-        Hash.new.tap do |h|
-          h.merge!(family_name: style.family_name,
-                   type: style.style,
-                   full_name: style.full_name)
-          if style.preferred_family_name
-            h[:preferred_family_name] = style.preferred_family_name
-          end
-          h[:preferred_type] = style.preferred_style if style.preferred_style
-          h.merge!(style.to_h.select do |k, _|
-            %i(post_script_name version description copyright).include?(k)
-          end.compact)
-          h.merge!(font: fix_variable_filename(style.filename))
-        end
-      end
-
-      def fix_variable_filename(filename)
-        filename.sub("[wght]", "-VariableFont_wght")
-      end
-
-      def formula_path(name)
-        Fontist::Import::Google.formula_path(name)
-      end
-
-      def save_to_path(hash, path)
-        File.write(path, YAML.dump(hash))
+      def rebuild_index
+        Fontist::Index.rebuild
       end
     end
   end