clip-rb 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 29e8df54de90f811ae779c79b8430d07cf7f23cf0d1995ccb9d3dcd268357df2
+  data.tar.gz: 556ac48aef1f241fb28e2907a9e5ae6b77930c57ccd67c6ba2e89d7f0a09d63a
+SHA512:
+  metadata.gz: af8f9334c49912107269621066863a22c393b54c13ed5464df7e7abd77803fc70a236bcf41bf5a1ad33ad5c6226e8741b5ae4198be0a2e84c117bb181306a2c8
+  data.tar.gz: 42db46f9e13d6792335607471c4250d272358da3e8fec24afdf23510da171bec9a88b0853575abdb53196676641f3a7da7141c62cd67656d6071a3f7769f00d9

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,132 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, caste, color, religion, or sexual
+identity and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming,
+diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment for our
+community include:
+
+* Demonstrating empathy and kindness toward other people
+* Being respectful of differing opinions, viewpoints, and experiences
+* Giving and gracefully accepting constructive feedback
+* Accepting responsibility and apologizing to those affected by our mistakes,
+  and learning from the experience
+* Focusing on what is best not just for us as individuals, but for the overall
+  community
+
+Examples of unacceptable behavior include:
+
+* The use of sexualized language or imagery, and sexual attention or advances of
+  any kind
+* Trolling, insulting or derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or email address,
+  without their explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Enforcement Responsibilities
+
+Community leaders are responsible for clarifying and enforcing our standards of
+acceptable behavior and will take appropriate and fair corrective action in
+response to any behavior that they deem inappropriate, threatening, offensive,
+or harmful.
+
+Community leaders have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct, and will communicate reasons for moderation
+decisions when appropriate.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when
+an individual is officially representing the community in public spaces.
+Examples of representing our community include using an official email address,
+posting via an official social media account, or acting as an appointed
+representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported to the community leaders responsible for enforcement at
+[INSERT CONTACT METHOD].
+All complaints will be reviewed and investigated promptly and fairly.
+
+All community leaders are obligated to respect the privacy and security of the
+reporter of any incident.
+
+## Enforcement Guidelines
+
+Community leaders will follow these Community Impact Guidelines in determining
+the consequences for any action they deem in violation of this Code of Conduct:
+
+### 1. Correction
+
+**Community Impact**: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community.
+
+**Consequence**: A private, written warning from community leaders, providing
+clarity around the nature of the violation and an explanation of why the
+behavior was inappropriate. A public apology may be requested.
+
+### 2. Warning
+
+**Community Impact**: A violation through a single incident or series of
+actions.
+
+**Consequence**: A warning with consequences for continued behavior. No
+interaction with the people involved, including unsolicited interaction with
+those enforcing the Code of Conduct, for a specified period of time. This
+includes avoiding interactions in community spaces as well as external channels
+like social media. Violating these terms may lead to a temporary or permanent
+ban.
+
+### 3. Temporary Ban
+
+**Community Impact**: A serious violation of community standards, including
+sustained inappropriate behavior.
+
+**Consequence**: A temporary ban from any sort of interaction or public
+communication with the community for a specified period of time. No public or
+private interaction with the people involved, including unsolicited interaction
+with those enforcing the Code of Conduct, is allowed during this period.
+Violating these terms may lead to a permanent ban.
+
+### 4. Permanent Ban
+
+**Community Impact**: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior, harassment of an
+individual, or aggression toward or disparagement of classes of individuals.
+
+**Consequence**: A permanent ban from any sort of public interaction within the
+community.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 2.1, available at
+[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1].
+
+Community Impact Guidelines were inspired by
+[Mozilla's code of conduct enforcement ladder][Mozilla CoC].
+
+For answers to common questions about this code of conduct, see the FAQ at
+[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at
+[https://www.contributor-covenant.org/translations][translations].
+
+[homepage]: https://www.contributor-covenant.org
+[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html
+[Mozilla CoC]: https://github.com/mozilla/diversity
+[FAQ]: https://www.contributor-covenant.org/faq
+[translations]: https://www.contributor-covenant.org/translations

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2025 Krzysztof Hasiński
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,66 @@
+# clip-rb
+
+[![Gem Version](https://badge.fury.io/rb/clip-rb.svg)](https://badge.fury.io/rb/clip-rb)
+[![Test](https://github.com/khasinski/clip-rb/workflows/clip-rb/badge.svg)](https://github.com/khasinski/clip-rb/actions/workflows/main.yml)
+
+Clip replacement that uses ONNX models. No Python required! 
+
+## Requirements
+
+- Ruby 3.0.0 or later
+- ONNX models for CLIP (downloaded automatically on first use)
+
+## Installation
+
+Install the gem and add to the application's Gemfile by executing:
+
+```bash
+bundle add clip-rb
+```
+
+If bundler is not being used to manage dependencies, install the gem by executing:
+
+```bash
+gem install clip-rb
+```
+
+## Usage
+
+```ruby
+require 'clip'
+
+clip = Clip::Model.new
+
+clip.encode_text("a photo of a cat") # => [0.15546110272407532, 0.07329428941011429, ...]
+
+clip.encode_image("test/fixtures/test.jpg") # => [0.22115306556224823,, 0.19343754649162292, ...]
+```
+
+## CLI
+
+Additionally you can fetch embeddings by calling:
+
+```bash
+$ clip-embed-text "a photo of a cat"
+$ clip-embed-image test/fixtures/test.jpg
+```
+
+Use KNN vector search to find similar images, remember to use cosine distance!
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/clip-rb. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [code of conduct](https://github.com/[USERNAME]/clip-rb/blob/main/CODE_OF_CONDUCT.md).
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+
+## Code of Conduct
+
+Everyone interacting in the clip-rb project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/[USERNAME]/clip-rb/blob/main/CODE_OF_CONDUCT.md).

data/Rakefile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec

data/exe/clip-embed-image

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+
+require_relative "../lib/clip"
+
+
+if ARGV.length != 1
+  puts "Usage: clip-embed-image <image_file>"
+  exit 1
+end
+
+begin
+  puts Clip::Model.new.encode_image(ARGV[0])
+rescue => e
+  puts "Error: #{e.message}"
+  exit 1
+end

data/exe/clip-embed-text

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+
+require_relative "../lib/clip"
+
+
+if ARGV.length != 1
+  puts "Usage: clip-embed-text <text>"
+  exit 1
+end
+
+begin
+  puts Clip::Model.new.encode_text(ARGV[0])
+rescue => e
+  puts "Error: #{e.message}"
+  exit 1
+end

data/lib/clip/image_preprocessor.rb

@@ -0,0 +1,5 @@
+module Clip
+  class ImagePreprocessor
+
+  end
+end

data/lib/clip/model.rb

@@ -0,0 +1,42 @@
+require "onnxruntime"
+
+module Clip
+  class Model
+    def initialize(
+      textual_model_path: ".clip_models/textual.onnx",
+      visual_model_path: ".clip_models/visual.onnx",
+      tokenizer: Clip::Tokenizer.new,
+      image_preprocessor: Clip::ImagePreprocessor.new,
+      download_models: true,
+      download_dir: ".clip_models"
+    )
+      @textual_model_path = textual_model_path
+      @visual_model_path = visual_model_path
+      Clip.download_models(download_dir) if download_models && !Clip.models_exist?(textual_model_path: textual_model_path, visual_model_path: visual_model_path)
+      @tokenizer = tokenizer
+      @image_preprocessor = image_preprocessor
+    end
+
+    def encode_text(text)
+      tokens = tokenizer.encode(text)
+      text_model.predict({ input: [ tokens ] })["output"].first
+    end
+
+    def encode_image(image)
+      image = image_preprocessor.preprocess(image).to_a
+      image_model.predict({ input: [ image ] })["output"].first
+    end
+
+    def text_model
+      @text_model ||= OnnxRuntime::Model.new(textual_model_path)
+    end
+
+    def image_model
+      @image_model ||= OnnxRuntime::Model.new(visual_model_path)
+    end
+
+    private
+
+    attr_reader :textual_model_path, :visual_model_path, :tokenizer, :image_preprocessor
+  end
+end

data/lib/clip/tokenizer.rb

@@ -0,0 +1,143 @@
+require "zlib"
+require "set"
+
+module Clip
+  class Tokenizer
+    INPUT_VECTOR_SIZE = 77
+
+    def initialize(bpe_path = __dir__ + "/../bpe_simple_vocab_16e6.txt.gz")
+      @byte_encoder = bytes_to_unicode
+      @byte_decoder = @byte_encoder.invert
+      merges = Zlib::GzipReader.open(bpe_path).read.split("\n")[1..(49152 - 256 - 2)]
+      merges = merges.map { |merge| merge.split(" ") }
+      vocab = @byte_encoder.values
+      vocab += vocab.map { |v| "#{v}</w>" }
+      merges.each { |merge| vocab << merge.join }
+      vocab += [ "<|startoftext|>", "<|endoftext|>" ]
+      @encoder = Hash[vocab.zip(0...vocab.size)]
+      @decoder = @encoder.invert
+      @bpe_ranks = Hash[merges.zip(0...merges.size)]
+      @cache = { "<|startoftext|>" => "<|startoftext|>", "<|endoftext|>" => "<|endoftext|>" }
+      @pattern = Regexp.new("<\\|startoftext\\|>|<\\|endoftext\\|>|'s|'t|'re|'ve|'m|'ll|'d|\\p{L}+|\\p{N}|[^\\s\\p{L}\\p{N}]+", Regexp::IGNORECASE)
+    end
+
+    def bytes_to_unicode
+      # Define base ranges for printable ASCII and extended Unicode
+      bs = (33..126).to_a + (161..172).to_a + (174..255).to_a # Printable characters
+      cs = bs.dup # Start with the same set of characters for mapping
+
+      # Map remaining bytes (0–255) to unique Unicode codepoints starting from 256
+      n = 0
+      (0...256).each do |b|
+        unless bs.include?(b)
+          bs << b                   # Add the byte to the mapping
+          cs << (256 + n)           # Assign a unique Unicode codepoint
+          n += 1                    # Increment the counter for unmapped bytes
+        end
+      end
+
+      # Convert codepoints to UTF-8 strings
+      cs = cs.map { |n| n.chr(Encoding::UTF_8) }
+
+      # Create a hash mapping bytes (0–255) to Unicode strings
+      Hash[bs.zip(cs)]
+    end
+
+    def get_pairs(word)
+      pairs = Set.new
+      prev_char = word[0]
+      word[1..-1].each do |char|
+        pairs.add([ prev_char, char ])
+        prev_char = char
+      end
+      pairs
+    end
+
+    def basic_clean(text)
+      text
+    end
+
+    def whitespace_clean(text)
+      text.gsub(/\s+/, " ").strip
+    end
+
+    def bpe(token)
+      return @cache[token] if @cache.key?(token)
+
+      word = token.chars[0..-2] + [ "#{token[-1]}</w>" ]
+      pairs = get_pairs(word)
+
+      until pairs.empty?
+        bigram = pairs.min_by { |pair| @bpe_ranks.fetch(pair, Float::INFINITY) }
+        break unless @bpe_ranks.key?(bigram)
+
+        first, second = bigram
+        new_word = []
+        i = 0
+        while i < word.size
+          j = word[i..-1]&.index(first)
+          j = j.nil? ? nil : j + i
+
+          if j.nil?
+            new_word.concat(word[i..-1])
+            break
+          else
+            new_word.concat(word[i...j])
+            if word[j] == first && word[j + 1] == second
+              new_word << "#{first}#{second}"
+              i = j + 2
+            else
+              new_word << word[j]
+              i = j + 1
+            end
+          end
+        end
+
+        word = new_word
+        break if word.size == 1
+
+        pairs = get_pairs(word)
+      end
+
+      result = word.join(" ")
+      @cache[token] = result
+      result
+    end
+
+    def encode(text)
+      bpe_tokens = []
+      cleaned_text = whitespace_clean(basic_clean(text)).downcase
+      cleaned_text = "<|startoftext|>#{cleaned_text}<|endoftext|>"
+      cleaned_text.scan(@pattern) do |token|
+        utf8_bytes = token.encode("utf-8").bytes
+
+        mapped_chars = utf8_bytes.map do |b|
+          @byte_encoder[b]
+        end
+        encoded = mapped_chars.join
+
+        bpe_subtokens = bpe(encoded).split(" ")
+
+        bpe_subtokens.each do |subtok|
+          bpe_tokens << @encoder[subtok]
+        end
+      end
+      pad_array(bpe_tokens)
+    end
+
+    def decode(tokens)
+      text = tokens.map { |token| @decoder[token] }.join
+      text = text.gsub("</w>", " ")
+
+      decoded_bytes = text.each_char.map do |c|
+        @byte_decoder[c]
+      end
+
+      decoded_bytes.compact.pack("C*").force_encoding("utf-8")
+    end
+
+    def pad_array(array)
+      array.fill(0, array.length...INPUT_VECTOR_SIZE).first(INPUT_VECTOR_SIZE)
+    end
+  end
+end

data/lib/clip/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Clip
+  VERSION = "0.1.0"
+end

data/lib/clip.rb

@@ -0,0 +1,50 @@
+require "net/http"
+require "fileutils"
+require_relative "clip/model"
+require_relative "clip/tokenizer"
+require_relative "clip/image_preprocessor"
+require "logger"
+
+module Clip
+  attr_accessor :logger
+
+  BASE_URL = "https://huggingface.co/khasinski/"
+  MODELS = {
+    textual: "clip-ViT-B-32-onnx/resolve/main/textual.onnx?download=true",
+    visual: "clip-ViT-B-32-onnx/resolve/main/visual.onnx?download=true"
+  }
+
+  def self.download_models(download_dir)
+    logger ||= Logger.new(STDOUT)
+    FileUtils.mkdir_p(download_dir)
+
+    MODELS.each do |type, path|
+      uri = URI.join(BASE_URL, path)
+      logger.info("Downloading #{type} model from #{uri}")
+
+      while true
+        response = Net::HTTP.get_response(uri)
+
+        if response.is_a?(Net::HTTPRedirection)
+          logger.info("Redirected to #{response['location']}")
+          uri = URI.parse(response['location']) # Update URI to the redirect location
+          next
+        elsif response.is_a?(Net::HTTPSuccess)
+          file_path = File.join(download_dir, "#{type}.onnx")
+          File.open(file_path, 'wb') do |file|
+            file.write(response.body) # Write the body directly for simplicity
+          end
+          logger.info("Successfully downloaded #{type} model")
+          break
+        else
+          logger.error("Failed to download #{type} model from #{uri}: #{response.code} #{response.message}")
+          raise "Failed to download #{type} model from #{uri}"
+        end
+      end
+    end
+  end
+
+  def self.models_exist?(textual_model_path:, visual_model_path:)
+    File.exist?(textual_model_path) && File.exist?(visual_model_path)
+  end
+end

data/sig/clip/rb.rbs

@@ -0,0 +1,6 @@
+module Clip
+  module Rb
+    VERSION: String
+    # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+  end
+end

metadata

@@ -0,0 +1,120 @@
+--- !ruby/object:Gem::Specification
+name: clip-rb
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Krzysztof Hasiński
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2025-01-27 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: onnxruntime
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.9.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.9.3
+- !ruby/object:Gem::Dependency
+  name: net-http
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.6.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.6.0
+- !ruby/object:Gem::Dependency
+  name: zlib
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+- !ruby/object:Gem::Dependency
+  name: logger
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+description: OpenAI CLIP embeddings, uses ONNX models. Allows to create embeddings
+  for images and text
+email:
+- krzysztof.hasinski@gmail.com
+executables:
+- clip-embed-image
+- clip-embed-text
+extensions: []
+extra_rdoc_files: []
+files:
+- ".clip_models/.gitkeep"
+- ".rspec"
+- CODE_OF_CONDUCT.md
+- LICENSE.txt
+- README.md
+- Rakefile
+- exe/clip-embed-image
+- exe/clip-embed-text
+- lib/bpe_simple_vocab_16e6.txt.gz
+- lib/clip.rb
+- lib/clip/image_preprocessor.rb
+- lib/clip/model.rb
+- lib/clip/tokenizer.rb
+- lib/clip/version.rb
+- sig/clip/rb.rbs
+homepage: https://github.com/khasinski/clip-rb
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/khasinski/clip-rb
+  source_code_uri: https://github.com/khasinski/clip-rb
+  changelog_uri: https://github.com/khasinski/clip-rb/CHANGELOG.md
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.0.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.5.22
+signing_key:
+specification_version: 4
+summary: OpenAI CLIP embeddings, uses ONNX models
+test_files: []