fontist 1.20.0 → 1.21.2

data/lib/fontist/import/formula_builder.rb

@@ -1,49 +1,70 @@
 require "shellwords"
 require_relative "text_helper"
+require_relative "helpers/hash_helper"
 
 module Fontist
   module Import
     class FormulaBuilder
-      FORMULA_ATTRIBUTES = %i[platforms description homepage resources
+      FORMULA_ATTRIBUTES = %i[name platforms description homepage resources
                               font_collections fonts extract copyright
                               license_url requires_license_agreement
                               open_license digest command].freeze
 
-      attr_writer :archive,
-                  :url,
-                  :extractor,
+      attr_writer :resources,
                   :options,
                   :font_files,
                   :font_collection_files,
                   :license_text,
-                  :homepage
+                  :operations
 
       def initialize
         @options = {}
+        @font_files = []
+        @font_collection_files = []
       end
 
       def formula
         formula_attributes.map { |name| [name, send(name)] }.to_h.compact
       end
 
+      def save
+        path = vacant_path
+        yaml = YAML.dump(Helpers::HashHelper.stringify_keys(formula))
+        File.write(path, yaml)
+        path
+      end
+
+      private
+
+      def formula_attributes
+        FORMULA_ATTRIBUTES
+      end
+
       def name
+        @name ||= generate_name
+      end
+
+      def generate_name
         return @options[:name] if @options[:name]
 
-        common = %i[family_name type]
-          .map { |attr| both_fonts.map(&attr).uniq }
-          .map { |names| TextHelper.longest_common_prefix(names) }
-          .map { |prefix| prefix unless prefix == "Regular" }
-          .compact
-          .join(" ")
+        common = common_prefix
         return common unless common.empty?
 
         both_fonts.map(&:family_name).first
       end
 
-      private
+      def common_prefix
+        family_prefix = common_prefix_by_attr(:family_name)
+        style_prefix = common_prefix_by_attr(:type)
 
-      def formula_attributes
-        FORMULA_ATTRIBUTES
+        [family_prefix, style_prefix].compact.join(" ")
+      end
+
+      def common_prefix_by_attr(attr)
+        names = both_fonts.map(&attr).uniq
+        prefix = TextHelper.longest_common_prefix(names)
+
+        prefix unless prefix == "Regular"
       end
 
       def both_fonts
@@ -70,69 +91,7 @@ module Fontist
       end
 
       def resources
-        filename = name.gsub(" ", "_") + "." + @extractor.extension
-
-        { filename => resource_options }
-      end
-
-      def resource_options
-        if @options[:skip_sha]
-          resource_options_without_sha
-        else
-          resource_options_with_sha
-        end
-      end
-
-      def resource_options_without_sha
-        { urls: [@url] + mirrors, file_size: file_size }
-      end
-
-      def resource_options_with_sha
-        urls = []
-        sha = []
-        downloads do |url, path|
-          urls << url
-          sha << Digest::SHA256.file(path).to_s
-        end
-
-        sha = prepare_sha256(sha)
-
-        { urls: urls, sha256: sha, file_size: file_size }
-      end
-
-      def downloads
-        yield @url, @archive
-
-        mirrors.each do |url|
-          path = download(url)
-          next unless path
-
-          yield url, path
-        end
-      end
-
-      def mirrors
-        @options[:mirror] || []
-      end
-
-      def download(url)
-        Fontist::Utils::Downloader.download(url, progress_bar: true).path
-      rescue Errors::InvalidResourceError
-        Fontist.ui.error("WARN: a mirror is not found '#{url}'")
-        nil
-      end
-
-      def prepare_sha256(input)
-        output = input.uniq
-        return output.first if output.size == 1
-
-        checksums = output.join(", ")
-        Fontist.ui.error("WARN: SHA256 differs (#{checksums})")
-        output
-      end
-
-      def file_size
-        File.size(@archive)
+        @resources || raise("Resources should be set.")
       end
 
       def font_collections
@@ -175,7 +134,7 @@ module Fontist
       end
 
       def extract
-        @extractor.operations
+        @operations || {}
       end
 
       def copyright
@@ -197,9 +156,11 @@ module Fontist
 
         return unless @license_text
 
-        Fontist.ui.error("WARN: ensure it's an open license, otherwise " \
-                         "change the 'open_license' attribute to " \
-                         "'requires_license_agreement'")
+        unless @options[:open_license]
+          Fontist.ui.error("WARN: ensure it's an open license, otherwise " \
+                           "change the 'open_license' attribute to " \
+                           "'requires_license_agreement'")
+        end
 
         TextHelper.cleanup(@license_text)
       end
@@ -211,6 +172,27 @@ module Fontist
       def command
         Shellwords.shelljoin(ARGV)
       end
+
+      def vacant_path
+        path = path_from_name
+        return path unless @options[:keep_existing] && File.exist?(path)
+
+        2.upto(9) do |i|
+          candidate = path.sub(/\.yml$/, "#{i}.yml")
+          return candidate unless File.exist?(candidate)
+        end
+
+        raise Errors::GeneralError, "Formula #{path} already exists."
+      end
+
+      def path_from_name
+        filename = Import.name_to_filename(name)
+        if @options[:formula_dir]
+          File.join(@options[:formula_dir], filename)
+        else
+          filename
+        end
+      end
     end
   end
 end

data/lib/fontist/import/google/api.rb

@@ -0,0 +1,25 @@
+module Fontist
+  module Import
+    module Google
+      class Api
+        class << self
+          def items
+            db["items"]
+          end
+
+          def db
+            @db ||= JSON.parse(Net::HTTP.get(URI(url)))
+          end
+
+          def url
+            "https://www.googleapis.com/webfonts/v1/webfonts?key=#{api_key}"
+          end
+
+          def api_key
+            Fontist.google_fonts_key
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fontist/import/google/create_google_formula.rb

@@ -0,0 +1,89 @@
+require "fontist/import"
+require "fontist/import/formula_builder"
+require "fontist/import/otf/font_file"
+
+module Fontist
+  module Import
+    module Google
+      class CreateGoogleFormula
+        REPO_PATH = Fontist.fontist_path.join("google", "fonts")
+        POSSIBLE_LICENSE_FILES = ["LICENSE.txt",
+                                  "LICENCE.txt",
+                                  "OFL.txt",
+                                  "UFL.txt"].freeze
+
+        def initialize(item, options = {})
+          @item = item
+          @options = options
+        end
+
+        def call
+          builder = FormulaBuilder.new
+          builder.options = options
+          builder.resources = resources
+          builder.font_files = font_files
+          builder.license_text = license_text
+          builder.save
+        end
+
+        private
+
+        def options
+          @options.merge(name: formula_name, open_license: true)
+        end
+
+        def formula_name
+          @item["family"]
+        end
+
+        def resources
+          {
+            @item["family"] => {
+              source: "google",
+              family: @item["family"],
+              files: @item["files"].values,
+            },
+          }
+        end
+
+        def font_files
+          @font_files ||= @item["files"].map do |_key, url|
+            font_file(url)
+          end
+        end
+
+        def license_text
+          @license_text ||= find_license_text
+        end
+
+        def font_file(url)
+          path = Utils::Downloader.download(url, use_content_length: false).path
+          Otf::FontFile.new(path)
+        end
+
+        def find_license_text
+          file = license_file
+          return unless file
+
+          File.read(file)
+        end
+
+        def license_file
+          dir = @item["family"].gsub(" ", "").downcase
+          path = repo_paths(dir).first
+          return unless path
+
+          full_paths = POSSIBLE_LICENSE_FILES.map { |f| File.join(path, f) }
+
+          Dir[*full_paths].first
+        end
+
+        def repo_paths(dir)
+          Dir[File.join(REPO_PATH, "apache", dir),
+              File.join(REPO_PATH, "ofl", dir),
+              File.join(REPO_PATH, "ufl", dir)]
+        end
+      end
+    end
+  end
+end

data/lib/fontist/import/google_import.rb

@@ -1,54 +1,92 @@
-require "erb"
 require_relative "google"
-require_relative "google/new_fonts_fetcher"
-require_relative "create_formula"
+require_relative "google/api"
+require_relative "google/create_google_formula"
 
 module Fontist
   module Import
     class GoogleImport
+      REPO_PATH = Fontist.fontist_path.join("google", "fonts")
+      REPO_URL = "https://github.com/google/fonts.git".freeze
+
       def initialize(options)
         @max_count = options[:max_count] || Google::DEFAULT_MAX_COUNT
       end
 
       def call
-        fonts = new_fonts
-        create_formulas(fonts)
-        rebuild_index unless fonts.empty?
+        update_repo
+        count = update_formulas
+        rebuild_index if count.positive?
       end
 
       private
 
-      def new_fonts
-        Fontist::Import::Google::NewFontsFetcher.new(logging: true,
-                                                     limit: @max_count).call
+      def update_repo
+        if Dir.exist?(REPO_PATH)
+          `cd #{REPO_PATH} && git pull`
+        else
+          FileUtils.mkdir_p(File.dirname(REPO_PATH))
+          `git clone --depth 1 #{REPO_URL} #{REPO_PATH}`
+        end
       end
 
-      def create_formulas(fonts)
-        return puts("Nothing to update") if fonts.empty?
+      def update_formulas
+        Fontist.ui.say "Updating formulas..."
+
+        items = api_items
 
-        puts "Creating formulas..."
-        fonts.each do |path|
-          create_formula(path)
+        count = 0
+        items.each do |item|
+          break if count >= @max_count
+
+          path = update_formula(item)
+          count += 1 if path
         end
+
+        count
+      end
+
+      def api_items
+        Google::Api.items
+      end
+
+      def update_formula(item)
+        family = item["family"]
+        Fontist.ui.say "Checking #{family}"
+        unless new_changes?(item)
+          Fontist.ui.say "Skip, no changes"
+          return
+        end
+
+        create_formula(item)
       end
 
-      def create_formula(font_path)
-        puts font_path
+      def new_changes?(item)
+        formula = formula(item["family"])
+        return true unless formula
 
-        path = Fontist::Import::CreateFormula.new(
-          url(font_path),
-          name: Google.metadata_name(font_path),
+        item["files"].values != formula.resources.first.files
+      end
+
+      def formula(font_name)
+        path = formula_path(font_name)
+        Formula.new_from_file(path) if File.exist?(path)
+      end
+
+      def formula_path(name)
+        snake_case = name.downcase.gsub(" ", "_")
+        filename = "#{snake_case}.yml"
+        Fontist.formulas_path.join("google", filename)
+      end
+
+      def create_formula(item)
+        path = Google::CreateGoogleFormula.new(
+          item,
           formula_dir: formula_dir,
-          skip_sha: variable_style?(font_path),
-          digest: Google.digest(font_path),
         ).call
 
         Fontist.ui.success("Formula has been successfully created: #{path}")
-      end
 
-      def url(path)
-        name = Google.metadata_name(path)
-        "https://fonts.google.com/download?family=#{ERB::Util.url_encode(name)}"
+        path
       end
 
       def formula_dir
@@ -57,13 +95,6 @@ module Fontist
         end
       end
 
-      def variable_style?(path)
-        fonts = Dir.glob(File.join(path, "*.{ttf,otf}"))
-        fonts.any? do |font|
-          File.basename(font).match?(/\[(.+,)?(wght|opsz)\]/)
-        end
-      end
-
       def rebuild_index
         Fontist::Index.rebuild
       end

data/lib/fontist/import/recursive_extraction.rb

@@ -19,10 +19,6 @@ module Fontist
         save_operation_subdir
       end
 
-      def extension
-        fetch_extension(@archive)
-      end
-
       def font_files
         ensure_extracted
         @font_files
@@ -52,18 +48,6 @@ module Fontist
         @operations[:options][:fonts_sub_dir] = @subdir
       end
 
-      def fetch_extension(file)
-        File.extname(filename(file)).sub(/^\./, "")
-      end
-
-      def filename(file)
-        if file.respond_to?(:original_filename)
-          file.original_filename
-        else
-          File.basename(file)
-        end
-      end
-
       def ensure_extracted
         return if @extracted
 

data/lib/fontist/manifest/locations.rb

@@ -2,6 +2,8 @@ module Fontist
   module Manifest
     class Locations
       def initialize(manifest)
+        Fontist.ui.debug("Manifest: #{manifest}")
+
         @manifest = manifest
       end
 

data/lib/fontist/resources/archive_resource.rb

@@ -0,0 +1,55 @@
+module Fontist
+  module Resources
+    class ArchiveResource
+      def initialize(resource, options = {})
+        @resource = resource
+        @options = options
+      end
+
+      def files(_source_names, &block)
+        excavate.files(recursive_packages: true, &block)
+      end
+
+      private
+
+      def excavate
+        Excavate::Archive.new(archive.path)
+      end
+
+      def archive
+        download_file(@resource)
+      end
+
+      def download_file(source)
+        errors = []
+        source.urls.each do |request|
+          result = try_download_file(request, source)
+          return result unless result.is_a?(Errors::InvalidResourceError)
+
+          errors << result
+        end
+
+        raise Errors::InvalidResourceError, errors.join(" ")
+      end
+
+      def try_download_file(request, source)
+        info_log(request)
+
+        Fontist::Utils::Downloader.download(
+          request,
+          sha: source.sha256,
+          file_size: source.file_size,
+          progress_bar: !@options[:no_progress],
+        )
+      rescue Errors::InvalidResourceError => e
+        Fontist.ui.say(e.message)
+        e
+      end
+
+      def info_log(request)
+        url = request.respond_to?(:url) ? request.url : request
+        Fontist.ui.say(%(Downloading from #{url}))
+      end
+    end
+  end
+end

data/lib/fontist/resources/google_resource.rb

@@ -0,0 +1,64 @@
+module Fontist
+  module Resources
+    class GoogleResource
+      def initialize(resource, options = {})
+        @resource = resource
+        @options = options
+      end
+
+      def files(source_names)
+        cached_paths = download_fonts(source_names)
+
+        cached_paths.map do |path|
+          Dir.mktmpdir do |dir|
+            FileUtils.cp(path, dir)
+
+            yield File.join(dir, File.basename(path))
+          end
+        end
+      end
+
+      private
+
+      def download_fonts(source_names)
+        urls = font_urls(source_names)
+
+        urls.map do |url|
+          download(url)
+        end
+      end
+
+      def font_urls(source_names)
+        @resource.files.select do |url|
+          source_names.include?(path_to_source_file(url))
+        end
+      end
+
+      def path_to_source_file(path)
+        format_filename(File.basename(path))
+      end
+
+      # TODO: remove duplication, another in Cache
+      def format_filename(filename)
+        return filename unless filename.length > 255
+
+        ext = File.extname(filename)
+        target_size = 255 - ext.length
+        cut_filename = filename.slice(0, target_size)
+        "#{cut_filename}#{ext}"
+      end
+
+      def download(url)
+        Fontist.ui.say(%(Downloading from #{url}))
+
+        file = Utils::Downloader.download(
+          url,
+          use_content_length: false,
+          progress_bar: !@options[:no_progress],
+        )
+
+        file.path
+      end
+    end
+  end
+end

data/lib/fontist/style_version.rb

@@ -4,6 +4,10 @@ module Fontist
       @text = text
     end
 
+    def to_s
+      value.join(" . ")
+    end
+
     def value
       @value ||= numbers || default_value
     end

data/lib/fontist/utils/cache.rb

@@ -1,6 +1,8 @@
 module Fontist
   module Utils
     class Cache
+      MAX_FILENAME_SIZE = 255
+
       include Locking
 
       def self.lock_path(path)
@@ -110,6 +112,11 @@ module Fontist
       end
 
       def filename(source)
+        filename = response_to_filename(source)
+        format_filename(filename)
+      end
+
+      def response_to_filename(source)
         if File.extname(source.original_filename).empty? && source.content_type
           require "mime/types"
           ext = MIME::Types[source.content_type].first&.preferred_extension
@@ -119,6 +126,15 @@ module Fontist
         source.original_filename
       end
 
+      def format_filename(filename)
+        return filename unless filename.length > MAX_FILENAME_SIZE
+
+        ext = File.extname(filename)
+        target_size = MAX_FILENAME_SIZE - ext.length
+        cut_filename = filename.slice(0, target_size)
+        "#{cut_filename}#{ext}"
+      end
+
       def move(source_file, target_path)
         # Windows requires file descriptors to be closed before files are moved
         source_file.close

data/lib/fontist/utils/downloader.rb

@@ -10,12 +10,17 @@ module Fontist
         ruby2_keywords :download if respond_to?(:ruby2_keywords, true)
       end
 
-      def initialize(file, file_size: nil, sha: nil, progress_bar: nil)
+      def initialize(file,
+                     file_size: nil,
+                     sha: nil,
+                     progress_bar: nil,
+                     use_content_length: true)
         # TODO: If the first mirror fails, try the second one
         @file = file
         @sha = [sha].flatten.compact
         @file_size = file_size.to_i if file_size
         @progress_bar = progress_bar
+        @use_content_length = use_content_length
         @cache = Cache.new
       end
 
@@ -85,7 +90,9 @@ module Fontist
           max_redirects: 10,
           headers: headers,
           content_length_proc: ->(content_length) {
-            progress_bar.total = content_length if content_length
+            if @use_content_length && content_length
+              progress_bar.total = content_length
+            end
           },
           progress_proc: -> (progress) {
             progress_bar.increment(progress)