derived_images 0.3.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 6de1adc7c9f26d102288dc2d6eccc8581fbe3b0f8ae33934d5ffb3c0b4842b6d
+  data.tar.gz: 5f45a38c7566beaf40dc61861e4b29567add7ff87bcc12d14a5cb7bb53a6ee1f
+SHA512:
+  metadata.gz: 91401997195e143d4daa30e136d8b55e206f3b98f209488bc4a8545f457a6bf9391363d64d281678af5df2bd4c1a0ad047953e1c9fb5c7526eecdf276fb60f53
+  data.tar.gz: 5f24b1d1028eafcd016277b0c98ff26abd660c708675374e4360fb640ca12802efdf3412a77710ea0aea2c0a0c54836b4070cd2550f9a272a0df441f91db7a83

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright 2023 Michael Kitson
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,38 @@
+# DerivedImages
+
+ActiveStorage's `variant` functionality is awesome, so why not have it for images in the Rails asset pipeline too?
+
+Use DerivedImages to programmatically create image assets by applying transformations to other assets.
+Resize images, lower quality to save bytes, rotate, crop, convert between formats, and anything else that the
+[image_processing](https://rubygems.org/gems/image_processing) gem supports.
+
+## Installation
+
+1. Run `./bin/bundle add derived_images`
+2. Run `./bin/rails derived_images:install`
+3. Install Vips or ImageMagick (if not already installed for ActiveStorage/ImageProcessing)
+    - MacOS: `brew install vips` or `brew install imagemagick`
+    - Debian/Ubuntu: `apt install libvips42` or `apt install imagemagick`
+
+## Usage
+
+After installing, specify images to create in the config file at `config/derived_images.rb`.
+
+```ruby
+derive 'my_derived_image.webp', from: 'my_source_image.jpg'
+resize 'tiny.png', from: 'original.png', width: 400, height: 300
+derive 'fully_custom.jpg', from: 'original.jpg' do |pipeline|
+  pipeline.saver(quality: 50).resize_to_fill(80, 80).rotate(180)
+end
+```
+
+DerivedImages watches this file and the source images, compiling new assets into `app/assets/builds` where the asset
+pipeline can pick them up with the normal asset helpers.
+
+```erbruby
+<%= image_tag('tiny.png') %>
+```
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'bundler/gem_tasks'
+require 'rake/testtask'
+
+task default: :test
+Rake::TestTask.new do |t|
+  t.libs << 'test'
+  t.pattern = 'test/*_test.rb'
+  t.warning = false
+end

data/lib/derived_images/cache.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+module DerivedImages
+  # Build cache for derived image files.
+  class Cache
+    include Enumerable
+
+    # @param [Pathname, String, nil] path The cache directory, which will be created if missing. Nil disables caching.
+    def initialize(path = DerivedImages.config.cache_path)
+      @path = path && Pathname.new(path)
+    end
+
+    # Copy a cached file out to another path.
+    #
+    # @param [String] key Cache key
+    # @param [Pathname, String] path
+    def copy(key, path)
+      enabled? && FileUtils.copy(key_path(key), path)
+    end
+
+    # Check for the presence of a cached file.
+    #
+    # @param [String] key Cache key
+    # @return [Boolean]
+    def exist?(key)
+      enabled? && key_path(key).file?
+    end
+
+    # Remove a cached file, if it exists.
+    #
+    # @param [String] key Cache key
+    def remove(key)
+      return unless enabled?
+
+      key_path(key).delete if key_path(key).exist?
+      maybe_clean_dir_for(key)
+    end
+
+    # Copy a file into the cache.
+    #
+    # @param [String] key Cache key
+    # @param [Pathname, String] path
+    def store(key, path)
+      return unless enabled?
+
+      mkdir_for(key)
+      FileUtils.copy(path, key_path(key))
+    end
+
+    # Move (not copy) a file into the cache.
+    #
+    # @param [String] key Cache key
+    # @param [Pathname, String] path
+    def take_and_store(key, path)
+      return unless enabled?
+
+      mkdir_for(key)
+      FileUtils.mv(path, key_path(key))
+    end
+
+    # Convert a cache key into a filesystem path of where it would be stored.
+    #
+    # @param [String] key Cache key
+    # @return [Pathname]
+    def key_path(key)
+      path.join(key[0...2], key[2..])
+    end
+
+    # Iterates over cache keys.
+    #
+    # @yieldparam key [String] the cache key
+    # @return [DerivedImages::Cache, Enumerator]
+    def each
+      return enum_for(:each) unless block_given?
+
+      path.glob('*/*') { |path| yield path.to_s.last(65).delete('/') }
+      self
+    end
+
+    # Get the SHA256 hash of a cached file.
+    #
+    # @param [String] key Cache key
+    # @return [String, nil] The hex-encoded digest
+    def digest(key)
+      Digest::SHA256.file(key_path(key)).hexdigest if exist?(key)
+    end
+
+    private
+
+    attr_reader :path
+
+    # Create parent directories so that we can store a value at the given key.
+    #
+    # @param [String] key Cache key
+    def mkdir_for(key)
+      FileUtils.mkdir_p(key_path(key).dirname)
+    end
+
+    # Clean up any unnecessary parent directories above the given cache key.
+    #
+    # @param [String] key Cache key
+    def maybe_clean_dir_for(key)
+      dir = key_path(key).dirname
+      dir.rmdir if dir.directory? && dir.empty?
+    end
+
+    def enabled?
+      !@path.nil?
+    end
+  end
+end

data/lib/derived_images/dsl.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module DerivedImages
+  # Use these DSL functions in your `config/derived_images.rb` file to define images to derive.
+  #
+  # @note An image format conversions can be automatically inferred from the `target` file extension in these methods.
+  module Dsl
+    # Resize an image, preserving its aspect ratio.
+    #
+    # @param [String] target The relative file name of the target (derived) image
+    # @param [String] from The relative file name of the source image. Must be in one of the configured `image_paths`
+    # @param [Integer] width The max width of the derived image
+    # @param [Integer] height The max height of the derived image
+    # @return [ManifestEntry]
+    def resize(target, from:, width:, height:)
+      derive(target, from: from) { _1.resize_to_limit(width, height) }
+    end
+
+    # Derive one image from another, with full customization abilities.
+    #
+    # @param [String] target The relative file name of the target (derived) image
+    # @param [String] from The relative file name of the source image. Must be in one of the configured `image_paths`
+    # @yieldparam pipeline [ImageProcessing::Chainable] The pipeline you can use to further customize the transformation
+    # @return [ManifestEntry]
+    def derive(target, from:, &block)
+      pipeline = ManifestEntry.empty_pipeline
+      pipeline = yield(pipeline) if block
+      add_entry(ManifestEntry.new(from, target, pipeline))
+    end
+  end
+end

data/lib/derived_images/manifest.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module DerivedImages
+  # The Manifest creates and holds a list of {ManifestEntry} instances, describing every derived image to create.
+  class Manifest
+    include Dsl
+
+    def initialize(path = Rails.root.join(DerivedImages.config.manifest_path))
+      @path = path
+      @entry_map = {}
+    end
+
+    def draw(&block)
+      @entry_map.clear
+      if block
+        instance_eval(&block)
+      else
+        instance_eval(File.read(path), path.to_s)
+      end
+    end
+
+    def add_entry(entry)
+      entry_map[entry.target] = entry
+    end
+
+    delegate :[], :count, :each, :each_value, :filter_map, :key?, :length, to: :entry_map
+
+    def produced_from(source_path)
+      source_names = []
+      DerivedImages.config.image_paths.each do |path|
+        dir = Pathname.new(path).expand_path.realpath
+        contains_source_file = source_path.ascend.any? { _1 == dir }
+        source_names << source_path.relative_path_from(dir).to_s if contains_source_file
+      end
+      entry_map.filter_map { |_target, entry| source_names.include?(entry.source) ? entry : nil }
+    end
+
+    attr_reader :path
+
+    def diff_from(former_manifest)
+      changed = []
+      removed = []
+      former_manifest.each do |target, entry|
+        if key?(target)
+          changed << entry if entry_map[target] != entry
+        else
+          removed << entry
+        end
+      end
+      added = entry_map.filter_map { |target, entry| former_manifest.key?(target) ? nil : entry }
+      [added, changed, removed]
+    end
+
+    private
+
+    attr_reader :entry_map
+  end
+end

data/lib/derived_images/manifest_entry.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module DerivedImages
+  # A ManifestEntry describes how to create one derived image.
+  class ManifestEntry
+    attr_accessor :source, :target, :pipeline
+
+    def initialize(source, target, pipeline)
+      @source = source
+      @target = target
+      @pipeline = pipeline
+    end
+
+    def ==(other)
+      source == other.source && target == other.target && options_hash == other.options_hash
+    end
+
+    def source_path
+      DerivedImages.config.image_paths.each do |path|
+        path = Pathname.new(path).join(source).expand_path
+        return path if path.file?
+      end
+      nil
+    end
+
+    def target_path
+      Pathname.new(DerivedImages.config.build_path).join(target).expand_path
+    end
+
+    # Returns a cache key for the result of the image transformation. It will vary if the source file's content varies
+    # or if the operations applied to generate the target vary.
+    #
+    # @return [String, nil] A 64 character hexdigest, or nil if the source file can't be found
+    def cache_key
+      return nil unless source_present?
+
+      Digest::SHA256.hexdigest({ source: source_digest, pipeline: options_hash }.to_json)
+    end
+
+    def self.empty_pipeline
+      PROCESSORS.fetch(DerivedImages.config.processor).dup
+    end
+
+    PROCESSORS = { mini_magick: ImageProcessing::MiniMagick, vips: ImageProcessing::Vips }.freeze
+
+    def target_digest
+      Digest::SHA256.file(target_path).hexdigest
+    end
+
+    def source_present?
+      source_path&.file?
+    end
+
+    private
+
+    def source_digest
+      Digest::SHA256.file(source_path).hexdigest
+    end
+
+    protected
+
+    def options_hash
+      pipeline.branch.options
+    end
+  end
+end

data/lib/derived_images/processor.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module DerivedImages
+  # The Processor manages a {Manifest}, watches the filesystem for changes, and manages a pool of {Worker} instances
+  # which perform the image tasks.
+  class Processor
+    def initialize
+      @cache = Cache.new
+      @manifest = Manifest.new.tap(&:draw)
+      @queue = Thread::Queue.new
+      @workers = ThreadGroup.new
+    end
+
+    def watch
+      watch_manifest
+      watch_images
+      process_all
+    end
+
+    def unwatch
+      manifest_listener&.stop
+      image_listener&.stop
+    end
+
+    def run_once
+      process_all
+      queue.close
+    end
+
+    private
+
+    attr_reader :cache, :image_listener, :manifest, :manifest_listener, :queue, :workers
+
+    def watch_manifest
+      dir, file = manifest.path.split.map(&:to_s)
+      @manifest_listener = Listen.to(dir, only: Regexp.new(file)) do
+        DerivedImages.config.logger.debug('Reloading changed manifest')
+        @manifest = Manifest.new.tap(&:draw)
+        process_all
+      end
+      manifest_listener.start
+    end
+
+    def watch_images
+      @image_listener = Listen.to(*DerivedImages.config.image_paths) do |modified, added, removed|
+        (modified + added + removed).each do |path|
+          source_path = Pathname.new(path).expand_path.realpath
+          manifest.produced_from(source_path).each { enqueue(_1) }
+        end
+        prune_cache
+      end
+      image_listener.start
+    end
+
+    def process_all
+      manifest.each_value { enqueue(_1) }
+      prune_cache
+    end
+
+    def enqueue(entry)
+      should_expand = queue.num_waiting.zero? && workers.list.length < DerivedImages.config.threads
+      queue << entry
+      Worker.start(workers, queue) if should_expand
+    end
+
+    def prune_cache
+      expected_keys = manifest.filter_map { |_target, entry| entry.cache_key }
+      (cache.to_a - expected_keys).each do |key|
+        DerivedImages.config.logger.debug("Removing cached file at #{key}")
+        cache.remove(key)
+      end
+    end
+  end
+end

data/lib/derived_images/railtie.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module DerivedImages
+  # The Railtie runs the {Processor} as part of a rails server.
+  class Railtie < ::Rails::Railtie
+    config.derived_images = ActiveSupport::OrderedOptions.new.update(
+      build_path: 'app/assets/builds',
+      cache_path: Rails.env.development? ? 'tmp/cache/derived_images' : nil,
+      enabled?: Rails.env.development? || Rails.env.test?,
+      image_paths: ['app/assets/images'],
+      manifest_path: 'config/derived_images.rb',
+      processor: :vips,
+      threads: 5,
+      watch?: Rails.env.development?
+    )
+
+    initializer 'derived_images' do |app|
+      derived_images = app.config.derived_images
+      derived_images.logger = Rails.logger.tagged('derived_images')
+    end
+
+    rake_tasks do
+      load 'tasks/derived_images.rake'
+    end
+
+    server do |app|
+      derived_images = app.config.derived_images
+      next unless derived_images.enabled?
+
+      processor = DerivedImages::Processor.new
+      if derived_images.watch?
+        processor.watch
+      else
+        processor.run_once
+      end
+    end
+  end
+end

data/lib/derived_images/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module DerivedImages
+  VERSION = '0.3.0'
+end

data/lib/derived_images/worker.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module DerivedImages
+  # A Worker represents a thread in a thread pool that completes image creation tasks.
+  class Worker
+    def initialize(queue, cache = Cache.new)
+      @queue = queue
+      @cache = cache
+    end
+
+    def run
+      until queue.closed? && queue.empty?
+        entry = queue.pop
+        process(entry) if entry
+      end
+    end
+
+    def self.start(thread_group, queue)
+      DerivedImages.config.logger.debug('Starting a new worker thread')
+      thread_group.add(Thread.new { Worker.new(queue).run })
+    end
+
+    private
+
+    attr_reader :cache, :queue
+
+    def process(entry)
+      unless entry.source_present?
+        return DerivedImages.config.logger.error("Can't find #{entry.source} to build #{entry.target}")
+      end
+
+      cache_key = entry.cache_key
+      if cache.exist?(cache_key)
+        restore(entry, cache_key)
+      else
+        generate(entry, cache_key)
+      end
+    end
+
+    def restore(entry, cache_key)
+      return if entry.target_path.file? && cache.digest(cache_key) == entry.target_digest
+
+      cache.copy(cache_key, entry.target_path)
+      DerivedImages.config.logger.debug("Restored #{entry.target} from cache")
+    end
+
+    def generate(entry, cache_key)
+      time = Benchmark.realtime do
+        tempfile = entry.pipeline.loader(fail: true).call(entry.source_path.to_s)
+        FileUtils.mv(tempfile.path, entry.target_path)
+      end
+      cache.store(cache_key, entry.target_path)
+      DerivedImages.config.logger.info("Created #{entry.target} from #{entry.source} in #{time.round(3)}s")
+    end
+  end
+end

data/lib/derived_images.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require 'image_processing'
+require 'listen'
+
+require 'derived_images/cache'
+require 'derived_images/dsl'
+require 'derived_images/manifest'
+require 'derived_images/manifest_entry'
+require 'derived_images/processor'
+require 'derived_images/railtie'
+require 'derived_images/worker'
+require 'derived_images/version'
+
+# DerivedImages programmatically creates derived image assets by applying transformations to other assets.
+module DerivedImages
+  def self.config
+    Rails.application.config.derived_images
+  end
+end

data/lib/install/derived_images.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+# Define your derived images using the DSL in lib/derived_images/dsl.rb
+# derive "hero_1200.webp", from: "hero_1200.png"
+# resize "favicon-32x32.png", from: "favicon.svg", width: 32, height: 32

data/lib/install/install.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+say 'Create derived_images.rb manifest'
+copy_file "#{__dir__}/derived_images.rb", 'config/derived_images.rb'
+
+say 'Compile into app/assets/builds'
+empty_directory 'app/assets/builds'
+keep_file 'app/assets/builds'
+
+if (sprockets_manifest_path = Rails.root.join('app/assets/config/manifest.js')).exist?
+  append_to_file sprockets_manifest_path, %(//= link_tree ../builds\n)
+end

data/lib/tasks/derived_images.rake

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+namespace :derived_images do
+  desc 'install derived_images'
+  task :install do
+    system "#{RbConfig.ruby} ./bin/rails app:template LOCATION=#{File.expand_path('../install/install.rb',
+                                                                                  __dir__)}"
+  end
+
+  desc 'build assets from derived_images once'
+  task :build do
+    DerivedImages::Processor.new.run_once
+  end
+end
+
+Rake::Task['assets:precompile'].enhance(['derived_images:build']) if Rake::Task.task_defined?('assets:precompile')

metadata

@@ -0,0 +1,106 @@
+--- !ruby/object:Gem::Specification
+name: derived_images
+version: !ruby/object:Gem::Version
+  version: 0.3.0
+platform: ruby
+authors:
+- Michael Kitson
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2023-05-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: image_processing
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: listen
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: railties
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '6.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '6.1'
+description: |-
+  Resize images, lower quality to save bytes, rotate, crop, convert between formats, and anything \
+  else that the image_processing gem supports.
+email:
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- MIT-LICENSE
+- README.md
+- Rakefile
+- lib/derived_images.rb
+- lib/derived_images/cache.rb
+- lib/derived_images/dsl.rb
+- lib/derived_images/manifest.rb
+- lib/derived_images/manifest_entry.rb
+- lib/derived_images/processor.rb
+- lib/derived_images/railtie.rb
+- lib/derived_images/version.rb
+- lib/derived_images/worker.rb
+- lib/install/derived_images.rb
+- lib/install/install.rb
+- lib/tasks/derived_images.rake
+homepage: https://github.com/michaelkitson/derived_images
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/michaelkitson/derived_images
+  source_code_uri: https://github.com/michaelkitson/derived_images
+  changelog_uri: https://github.com/michaelkitson/derived_images/releases
+  rubygems_mfa_required: 'true'
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.7.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.4.13
+signing_key:
+specification_version: 4
+summary: Programmatically create derived image assets by applying transformations
+  to other assets.
+test_files: []