jekyll_ai_related_posts 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 8b59977a0c1d912e06792f7123c7d443547f1decb9fdf4042d90a4fcd4e1eb4e
+  data.tar.gz: 159db1306777a201cff3a9d3532ddba444164dbfb780207fb9457ce4898df5f8
+SHA512:
+  metadata.gz: e0f73857997bdacd22059c542a1d2a642a3ea76aa165241c155a0e8b2cafdc4f48607f334a2ed7a265a252e86969d9e523bf4ef2ac5889e1f39c36d1a27792f5
+  data.tar.gz: a5f5f573459deb7fab308b1fb98a1d60ce1aeda95e28860161ed886a3ab90c43f1b7d36fea6cc5c570b05cd15284a59240e8676f5b10f44290b05fbacbd379df

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,5 @@
+# Omakase Ruby styling for Rails
+inherit_gem:
+  rubocop-rails-omakase: rubocop.yml
+
+# Your own specialized rules go here

data/CHANGELOG.md

@@ -0,0 +1,5 @@
+## [Unreleased]
+
+## [0.1.0] - 2024-04-18
+
+- Initial release

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2024 Mike Kasberg
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,101 @@
+# Jekyll AI Related Posts 🪄
+
+Jekyll ships with functionality that populates
+[related_posts](https://jekyllrb.com/docs/variables/) with the ten most recent
+posts. If you install
+[classifier_reborn](https://jekyll.github.io/classifier-reborn/) and use the
+`--lsi` option, Jekyll will populate `related_posts` using latent semantic
+indexing. 
+
+**Using AI is a much better approach.** Latent semantic indexing seems
+promising, but in practice requires libraries like Numo or GSL that are tricky
+to install, and still produces mediocre results. In contrast, OpenAI offers an
+embeddings API that allows us to easily get the embedding vector (in one of
+OpenAI's models) of some text. We can use these vectors to compute related
+posts with the accuracy of OpenAI's models (or any other LLM, for that matter).
+
+## Installation
+
+Jekyll AI Related Posts is a [Jekyll
+plugin](https://jekyllrb.com/docs/plugins/installation/). It can be installed
+using any Jekyll plugin installation method.
+
+## Configuration
+
+All config for this plugin sits under a top-level `ai_related_posts` key.
+
+The only required config is `openai_api_key` -- we need to authenticate to the
+API to fetch embedding vectors.
+
+- **openai_api_key** Your OpenAI API key, used to fetch embeddings.
+- **fetch_enabled** (optional, default `true`). If true, fetch embeddings. If
+  false, don't fetch embeddings. If this is a string (like `prod`), fetch
+  embeddings only when the `JEKYLL_ENV` environment variable is equal to the
+  string. (This is useful if you want to reduce API costs by only fetching
+  embeddings on production builds.)
+
+### Example Config
+
+```yaml
+ai_related_posts:
+  openai_api_key: sk-proj-abc123
+  fetch_enabled: prod
+```
+
+## Usage
+
+When the plugin is installed and configured, it will populate an
+`ai_related_posts` key in the post data for all posts. Here's an example of how
+to use it:
+
+```liquid
+<h2>Related Posts</h2>
+<ul>
+  {% for post in page.ai_related_posts limit:3 %}
+    <li><a href="{{ post.url }}">{{ post.title }}</a></li>
+  {% endfor %}
+</ul>
+```
+
+### Upgrading from Built-In Related Posts
+
+If you're already using Jekyll's built-in `site.related_posts` and you want to
+upgrade to AI related posts:
+
+- Install the plugin.
+- Replace `site.related_posts` with `page.ai_related_posts` in your templates.
+- If you were using LSI, stop. It's no longer necessary. Don't pass the `--lsi`
+  option to the `jekyll` command. You can remove the `classifier-reborn` gem and
+  its dependencies (Numo).
+
+
+## How It Works
+
+Jekyll AI Related Posts is implemented as a Jekyll Generator plugin. During the
+build process, the plugin will call the [OpenAI Embeddings
+API](https://platform.openai.com/docs/guides/embeddings) to fetch the vector
+embedding for a string containing the title, tags, and categories of your
+article. It's not necessary to use the full post text, in most cases the title
+and tags produce very accurate results because the LLM knows when topics are
+related even if they never use identical words. This is also why the LLM
+produces better results than LSI. These vector embeddings are cached in a SQLite
+database. To query for related posts, we query the cached vectors using the
+[sqlite-vss](https://github.com/asg017/sqlite-vss) plugin.
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run
+`rake spec` to run the tests. You can also run `bin/console` for an interactive
+prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To
+release a new version, update the version number in `version.rb`, and then run
+`bundle exec rake release`, which will create a git tag for the version, push
+git commits and the created tag, and push the `.gem` file to
+[rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at
+https://github.com/mkasberg/jekyll_ai_related_posts.
+

data/Rakefile

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+require "rubocop/rake_task"
+
+RuboCop::RakeTask.new
+
+task default: %i[spec rubocop]

data/lib/jekyll_ai_related_posts/generator.rb

@@ -0,0 +1,181 @@
+# frozen_string_literal: true
+
+require "active_record"
+require "sqlite3"
+require "sqlite_vss"
+require "jekyll"
+require "json"
+
+module JekyllAiRelatedPosts
+  class Generator < Jekyll::Generator
+    def generate(site)
+      @site = site
+      setup_database
+
+      if fetch_enabled?
+        Jekyll.logger.info "[ai_related_posts] Generating related posts..."
+        @embeddings_fetcher = new_fetcher
+
+        @site.posts.docs.each do |p|
+          ensure_embedding_cached(p)
+        end
+
+        @indexed_posts = {}
+        site.posts.docs.each do |p|
+          @indexed_posts[p.relative_path] = p
+        end
+
+        @site.posts.docs.each do |p|
+          find_related(p)
+        end
+      else
+        Jekyll.logger.info "[ai_related_posts] Using cached related posts data..."
+
+        @site.posts.docs.each do |p|
+          fallback_generate_related(p)
+        end
+      end
+    end
+
+    private
+
+    def fetch_enabled?
+      enabled = true
+      if @site.config["ai_related_posts"]["fetch_enabled"].is_a? String
+        enabled = ENV["JEKYLL_ENV"] == @site.config["ai_related_posts"]["fetch_enabled"]
+      elsif [ true, false ].include? @site.config["ai_related_posts"]["fetch_enabled"]
+        enabled = @site.config["ai_related_posts"]["fetch_enabled"]
+      end
+
+      enabled
+    end
+
+    def fallback_generate_related(post)
+      existing = Models::Post.find_by(relative_path: post.relative_path)
+      if existing.nil?
+        post.data["ai_related_posts"] = post.related_posts
+      else
+        find_related(post)
+      end
+    end
+
+    def new_fetcher
+      case @site.config["ai_related_posts"]["embeddings_source"]
+      when "mock"
+        MockEmbeddings.new
+      else
+        OpenAiEmbeddings.new(@site.config["ai_related_posts"]["openai_api_key"])
+      end
+    end
+
+    def ensure_embedding_cached(post)
+      existing = Models::Post.find_by(relative_path: post.relative_path)
+
+      # Clear cache if post has been updated
+      if !existing.nil? && existing.embedding_text != embedding_text(post)
+        sql = "DELETE FROM vss_posts WHERE rowid = (SELECT rowid FROM posts WHERE relative_path = :relative_path);"
+        ActiveRecord::Base.connection.execute(ActiveRecord::Base.sanitize_sql([ sql,
+                                                                               { relative_path: post.relative_path } ]))
+        existing.destroy!
+        existing = nil
+      end
+
+      return unless existing.nil?
+
+      Models::Post.create!(
+        relative_path: post.relative_path,
+        embedding_text: embedding_text(post),
+        embedding: embedding_for(post).to_json
+      )
+
+      sql = <<-SQL
+          INSERT INTO vss_posts (rowid, post_embedding)
+            SELECT rowid, embedding FROM posts WHERE relative_path = :relative_path;
+      SQL
+      ActiveRecord::Base.connection.execute(ActiveRecord::Base.sanitize_sql([ sql,
+                                                                             { relative_path: post.relative_path } ]))
+    end
+
+    def find_related(post)
+      sql = <<-SQL
+        SELECT rowid, distance
+        FROM vss_posts
+        WHERE vss_search(
+          post_embedding,
+          (select embedding from posts where relative_path = :relative_path)
+        )
+        LIMIT 10000;
+      SQL
+
+      results = ActiveRecord::Base.connection.execute(ActiveRecord::Base.sanitize_sql([ sql, {
+                                                                                        relative_path: post.relative_path
+                                                                                      } ]))
+      # The first result is the post itself, with a distance of 0.
+      rowids = results.sort_by { |r| r["distance"] }.drop(1).first(3).map { |r| r["rowid"] }
+
+      posts_by_rowid = {}
+      rowids.each do |rowid|
+        # This *is* an N+1 query, but:
+        #  - N+1 penalty is way less with SQLite
+        #  - N is relatively small (it's Jekyll post count)
+        #  - This is an easy way to work around rowid not being a real column that ActiveRecord knows about.
+        posts_by_rowid[rowid] = Models::Post.select(:relative_path).find_by(rowid: rowid)
+      end
+
+      related_posts = rowids.map do |rowid|
+        relative_path = posts_by_rowid[rowid]["relative_path"]
+        @indexed_posts[relative_path]
+      end
+
+      post.data["ai_related_posts"] = related_posts
+    end
+
+    def embedding_text(post)
+      text = "Title: #{post.data["title"]}"
+      text += "; Categories: #{post.data["categories"].join(", ")}" unless post.data["categories"].empty?
+      text += "; Tags: #{post.data["tags"].join(", ")}" unless post.data["tags"].empty?
+
+      text
+    end
+
+    def embedding_for(post)
+      Jekyll.logger.info "[ai_related_posts] Fetching embedding for #{post.relative_path}"
+      input = embedding_text(post)
+
+      @embeddings_fetcher.embedding_for(input)
+    end
+
+    def setup_database
+      ActiveRecord::Base.establish_connection(
+        adapter: "sqlite3",
+        database: @site.in_source_dir(".ai_related_posts_cache.sqlite3")
+      )
+      # We don't need WAL mode for this.
+      ActiveRecord::Base.connection.execute("PRAGMA journal_mode=DELETE;")
+
+      # Enable sqlite-vss vector extension
+      db = ActiveRecord::Base.connection.raw_connection
+      db.enable_load_extension(true)
+      SqliteVss.load(db)
+      db.enable_load_extension(false)
+
+      create_posts = <<-SQL
+        CREATE TABLE IF NOT EXISTS posts(
+          relative_path TEXT PRIMARY KEY,
+          embedding_text TEXT,
+          embedding TEXT
+        );
+      SQL
+      ActiveRecord::Base.connection.execute(create_posts)
+
+      create_vss_posts = <<-SQL
+        CREATE VIRTUAL TABLE IF NOT EXISTS vss_posts using vss0(
+          post_embedding(#{OpenAiEmbeddings::DIMENSIONS})
+        );
+      SQL
+      ActiveRecord::Base.connection.execute(create_vss_posts)
+
+      Jekyll.logger.debug("ai_related_posts db setup complete")
+    end
+  end
+end

data/lib/jekyll_ai_related_posts/models/post.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module JekyllAiRelatedPosts
+  module Models
+    class Post < ActiveRecord::Base
+    end
+  end
+end

data/lib/jekyll_ai_related_posts/open_ai_embeddings.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require "faraday"
+
+module JekyllAiRelatedPosts
+  class OpenAiEmbeddings
+    DIMENSIONS = 1536
+
+    def initialize(api_key, connection: nil)
+      @connection = if connection.nil?
+                      Faraday.new(url: "https://api.openai.com") do |builder|
+                        builder.request :authorization, "Bearer", api_key
+                        builder.request :json
+                        builder.response :json
+                        builder.response :raise_error
+                      end
+      else
+                      connection
+      end
+    end
+
+    def embedding_for(text)
+      res = @connection.post("/v1/embeddings") do |req|
+        req.body = {
+          input: text,
+          model: "text-embedding-3-small"
+        }
+      end
+
+      res.body["data"].first["embedding"]
+    rescue Faraday::Error => e
+      Jekyll.logger.error "Error response from OpanAI API!"
+      Jekyll.logger.error e.inspect
+
+      raise
+    end
+  end
+end

data/lib/jekyll_ai_related_posts/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module JekyllAiRelatedPosts
+  VERSION = "0.1.0"
+end

data/lib/jekyll_ai_related_posts.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require_relative "jekyll_ai_related_posts/generator"
+
+require "zeitwerk"
+loader = Zeitwerk::Loader.for_gem
+loader.setup
+
+module JekyllAiRelatedPosts
+  GEM_ROOT = File.expand_path("..", __dir__)
+
+  class Error < StandardError; end
+end

metadata

@@ -0,0 +1,142 @@
+--- !ruby/object:Gem::Specification
+name: jekyll_ai_related_posts
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Mike Kasberg
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2024-04-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '7.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '7.1'
+- !ruby/object:Gem::Dependency
+  name: faraday
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.9'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.9'
+- !ruby/object:Gem::Dependency
+  name: jekyll
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
+- !ruby/object:Gem::Dependency
+  name: sqlite-vss
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.2
+- !ruby/object:Gem::Dependency
+  name: zeitwerk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.6'
+description: Populate ai_related_posts using Open AI embeddings
+email:
+- kasberg.mike@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".rspec"
+- ".rubocop.yml"
+- CHANGELOG.md
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/jekyll_ai_related_posts.rb
+- lib/jekyll_ai_related_posts/generator.rb
+- lib/jekyll_ai_related_posts/models/post.rb
+- lib/jekyll_ai_related_posts/open_ai_embeddings.rb
+- lib/jekyll_ai_related_posts/version.rb
+homepage: https://github.com/mkasberg/jekyll_ai_related_posts
+licenses:
+- MIT
+metadata:
+  allowed_push_host: https://rubygems.org
+  homepage_uri: https://github.com/mkasberg/jekyll_ai_related_posts
+  source_code_uri: https://github.com/mkasberg/jekyll_ai_related_posts
+  changelog_uri: https://github.com/mkasberg/jekyll_ai_related_posts
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.0.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.5.6
+signing_key:
+specification_version: 4
+summary: Populate ai_related_posts using Open AI embeddings
+test_files: []