jekyll-ai-related 0.1.0

data/jekyll-ai-related.gemspec

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require_relative "lib/jekyll/embeddings-generator/version"
+
+Gem::Specification.new do |spec|
+  spec.name = "jekyll-ai-related"
+  spec.version = Jekyll::EmbeddingsGenerator::VERSION
+  spec.authors = ["Francesco Pira"]
+  spec.email = ["dev@fpira.com"]
+
+  spec.summary = "Jekyll plugin to generate embeddings for posts and find related content"
+  spec.description = "A Jekyll plugin that uses OpenAI embeddings to analyze posts and find related content"
+  spec.homepage = "https://github.com/pirafrank/jekyll-ai-related"
+
+  spec.license = "MIT"
+  spec.required_ruby_version = ">= 3.2.0"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r!^bin/!) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.metadata["allowed_push_host"] = "https://rubygems.org"
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["changelog_uri"] = "https://github.com/pirafrank/jekyll-ai-related/blob/main/CHANGELOG.md"
+  spec.metadata["bug_tracker_uri"] = "https://github.com/pirafrank/jekyll-ai-related/issues"
+
+  spec.add_dependency "httparty", "~> 0.22.0"
+  spec.add_dependency "jekyll", ">= 3.7", "< 5.0"
+  spec.add_dependency "json", "~> 2.7"
+
+  spec.add_development_dependency "bundler", "~> 2.6"
+  spec.add_development_dependency "rake", "~> 13.0"
+  spec.add_development_dependency "rubocop-jekyll", "~> 0.14"
+end

data/lib/jekyll/commands/generator.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Jekyll
+  module Commands
+    class EmbeddingsGenerator < Command
+      class << self
+        def init_with_program(prog)
+          prog.command(:related) do |c|
+            c.description "Generate embeddings for each post and find related posts."
+            c.syntax "embeddings [options]"
+
+            c.option "debug",
+                     "--debug",
+                     "Most verbose. Set log level to Debug."
+            c.option "quiet",
+                     "--quiet",
+                     "Do not print Info logs. Set log level to Error."
+            c.option "future",
+                     "--future",
+                     "Get embeds and fine related posts also for those with a future date."
+            c.option "drafts",
+                     "--drafts",
+                     "Get embeds and find related posts also for drafts."
+
+            c.action do |_, opts|
+              Jekyll.logger.info "AI Related plugin starting..."
+              options = configuration_from_options(opts)
+              Jekyll::EmbeddingsGenerator.run(options)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/jekyll/embeddings-generator/embeddings/generate.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require "httparty"
+require "json"
+
+module Jekyll
+  module EmbeddingsGenerator
+    module Embeddings
+      class << self
+        include Jekyll::EmbeddingsGenerator
+
+        def generate_embeddings(text)
+          config = Jekyll::EmbeddingsGenerator.config
+          api_key = config["openai_api_key"]
+          response = HTTParty.post(
+            "https://api.openai.com/v1/embeddings",
+            :headers => {
+              "Authorization" => "Bearer #{api_key}",
+              "Content-Type"  => "application/json",
+            },
+            :body    => {
+              :model => "text-embedding-3-small",
+              :input => text,
+            }.to_json
+          )
+
+          raise "OpenAI API error: #{response.parsed_response["error"]["message"]}" unless response.success?
+
+          response.parsed_response["data"][0]["embedding"]
+        end
+      end
+    end
+  end
+end

data/lib/jekyll/embeddings-generator/embeddings/store.rb

@@ -0,0 +1,151 @@
+# frozen_string_literal: true
+
+require "httparty"
+require "json"
+
+module Jekyll
+  module EmbeddingsGenerator
+    module Store
+      class << self
+        include Jekyll::EmbeddingsGenerator
+
+        def store_embedding(data) # rubocop:disable Metrics/AbcSize
+          config = Jekyll::EmbeddingsGenerator.config
+          supabase_url = config["supabase_url"]
+          supabase_key = config["supabase_key"]
+
+          # First check if record exists and its edit date
+          existing = HTTParty.get(
+            "#{supabase_url}/rest/v1/page_embeddings",
+            :headers => {
+              "apikey"          => supabase_key,
+              "Authorization"   => "Bearer #{supabase_key}",
+              "Content-Type"    => "application/json",
+              "Accept-Encoding" => "identity", # this to avoid supabase returning gzipped content
+            },
+            :query   => {
+              "uid"    => "eq.#{data.uid}",
+              "select" => "uid, most_recent_edit",
+            }
+          )
+
+          Jekyll.logger.debug "response headers: #{existing.headers}"
+          Jekyll.logger.debug "response body: #{existing.body}"
+
+          raise "Supabase API error: #{existing.code} - #{existing.body}" unless existing.success?
+
+          existing_record = existing.parsed_response&.first
+          mre = data.most_recent_edit
+          should_update = existing_record.nil? || Time.parse(existing_record["most_recent_edit"]) < mre
+
+          false unless should_update
+
+          update_embedding(data)
+        end
+
+        def find_related(post)
+          config = Jekyll::EmbeddingsGenerator.config
+          post_uid = post.data[config["uid"]]
+          embedding = query_embeddings(post_uid)
+          find_related_posts(embedding, post_uid)
+        end
+
+        private
+
+        def update_embedding(data)
+          config = Jekyll::EmbeddingsGenerator.config
+          supabase_url = config["supabase_url"]
+          supabase_key = config["supabase_key"]
+
+          response = HTTParty.post(
+            "#{supabase_url}/rest/v1/page_embeddings",
+            :headers => {
+              "apikey"        => supabase_key,
+              "Authorization" => "Bearer #{supabase_key}",
+              "Content-Type"  => "application/json",
+              "Prefer"        => "resolution=merge-duplicates", # upsert behavior
+            },
+            :query   => {
+              "on_conflict" => "uid", # important: this MUST be declared as unique on database
+            },
+            :body    => {
+              :uid              => data.uid,
+              :most_recent_edit => data.most_recent_edit,
+              :embedding        => data.embedding,
+              :metadata         => data.metadata,
+              :content          => data.content,
+            }.to_json
+          )
+
+          return if response.success?
+
+          raise "Supabase API error: #{response.code} - #{response.body}"
+        end
+
+        def query_embeddings(post_uid)
+          config = Jekyll::EmbeddingsGenerator.config
+          supabase_url = config["supabase_url"]
+          supabase_key = config["supabase_key"]
+          response = HTTParty.get(
+            "#{supabase_url}/rest/v1/page_embeddings",
+            headers: {
+              "apikey"          => supabase_key,
+              "Authorization"   => "Bearer #{supabase_key}",
+              "Content-Type"    => "application/json",
+              "Accept-Encoding" => "identity", # this to avoid supabase returning gzipped content
+            },
+            query: {
+              "uid" => "eq.#{post_uid}",
+            }
+          )
+          Jekyll.logger.debug "response.parsed_response: #{response.parsed_response}"
+          raise "Supabase API error: #{response.code} - #{response.body}" unless response.success?
+
+          response.parsed_response.first&.dig("embedding")
+        end
+
+        def find_related_posts(embedding, post_uid)
+          config = Jekyll::EmbeddingsGenerator.config
+          supabase_url = config["supabase_url"]
+          supabase_key = config["supabase_key"]
+          score_threshold = config["score_threshold"]
+          limit = config["limit"] || 3
+          # Query using cosine similarity
+          # Note: this MUST be a stored procedure on Supabase, and order of
+          #       columns in 'select' statament must match the order of the
+          #       columns defined in the stored procedure.
+          query = %(
+                    select
+                      metadata->>'title' as title,
+                      uid as uid,
+                      most_recent_edit,
+                      metadata->>'url' as url,
+                      metadata->>'date' as date,
+                      1 - (embedding <=> '#{embedding}') as similarity
+                    from page_embeddings
+                    where uid != '#{post_uid}'
+                    and 1 - (embedding <=> '#{embedding}') > '#{score_threshold}'
+                    order by embedding <=> '#{embedding}'
+                    limit '#{limit}';
+                  )
+          response = HTTParty.post(
+            "#{supabase_url}/rest/v1/rpc/related_posts",
+            headers: {
+              "apikey"          => supabase_key,
+              "Authorization"   => "Bearer #{supabase_key}",
+              "Content-Type"    => "application/json",
+              "Accept-Encoding" => "identity", # this to avoid supabase returning gzipped content
+              "Prefer"          => "return=minimal",
+            },
+            body: {
+              query:,
+            }.to_json
+          )
+          raise "Supabase API error: #{response.code} - #{response.body}" unless response.success?
+
+          response.parsed_response
+        end
+      end
+    end
+  end
+end

data/lib/jekyll/embeddings-generator/init.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Jekyll
+  module EmbeddingsGenerator
+    class Configuration
+      include Jekyll::EmbeddingsGenerator
+
+      @config = {}
+
+      def self.init(opts) # rubocop:disable Metrics/AbcSize,Metrics/PerceivedComplexity
+        jk_config = Jekyll.configuration({})["jekyll-ai-related"] || {}
+        config = {}
+        config["uid"] = jk_config["post_unique_field"] || "slug"
+        config["mre"] = jk_config["post_updated_field"] || "date"
+        config["path"] = jk_config["output_path"] || "related_posts"
+        config["drafts"] = jk_config["include_drafts"] || opts["drafts"] || false
+        config["future"] = jk_config["include_future"] || opts["future"] || false
+        config["limit"] = jk_config["related_posts_limit"] || 3
+        config["score_threshold"] = jk_config["related_posts_score_threshold"] || 0.5
+        config["openai_api_key"] = ENV["OPENAI_API_KEY"]
+        config["supabase_url"] = ENV["SUPABASE_URL"]
+        config["supabase_key"] = ENV["SUPABASE_KEY"]
+        @config = config
+        validate
+        config
+      end
+
+      def self.build(options)
+        options["show_drafts"] = @config["drafts"]
+        options["future"] = @config["future"]
+        Jekyll.logger.debug "Show drafts? #{options["show_drafts"]}"
+        Jekyll.logger.debug "Include future posts? #{options["future"]}"
+        site = Jekyll::Site.new(options)
+        site.reset
+        site.read
+        # call the 'generate' method on all plugins inheriting from Jekyll::Generator.
+        # This allows to generate the site's content, including any additional data
+        # you may have added to the post objects via a custom plugin (which by default
+        # lives in the _plugins dir of you Jekyll installation).
+        site.generate
+        site
+      end
+
+      def self.validate
+        raise "Missing OpenAI API key" unless @config["openai_api_key"]
+        raise "Missing Supabase URL" unless @config["supabase_url"]
+        raise "Missing Supabase key" unless @config["supabase_key"]
+      end
+    end
+  end
+end

data/lib/jekyll/embeddings-generator/models/data.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require_relative "metadata"
+
+module Jekyll
+  module EmbeddingsGenerator
+    class Data
+      include Jekyll::EmbeddingsGenerator
+
+      attr_reader :uid, :most_recent_edit, :embedding, :metadata, :content
+
+      def initialize(post, embedding, metadata)
+        config = Jekyll::EmbeddingsGenerator.config
+        @uid = post.data[config["uid"]]
+        @most_recent_edit = post.data[config["mre"]]
+        @embedding = embedding
+        @metadata = metadata.to_h
+        @content = post.content
+      end
+
+      def to_h
+        {
+          :uid              => @uid,
+          :most_recent_edit => @most_recent_edit,
+          :embedding        => @embedding,
+          :metadata         => @metadata,
+          :content          => @content,
+        }.compact
+      end
+    end
+  end
+end

data/lib/jekyll/embeddings-generator/models/metadata.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Jekyll
+  module EmbeddingsGenerator
+    class Metadata
+      attr_reader :title, :subtitle, :description, :date, :slug, :uid,
+                  :url, :categories, :tags, :updates, :last_edit
+
+      def initialize(post) # rubocop:disable Metrics/AbcSize
+        @title = post.data["title"]
+        @subtitle = post.data["subtitle"]
+        @description = post.data["description"]
+        @date = post.data["date"]
+        @slug = post.data["slug"]
+        @uid = post.data["uid"]
+        @url = post.url
+        @categories = post.data["categories"]
+        @tags = post.data["tags"]
+        @updates = post.data["updates"]
+        @last_edit = post.data["most_recent_edit"]
+      end
+
+      def to_h
+        {
+          :title       => @title,
+          :subtitle    => @subtitle,
+          :description => @description,
+          :date        => @date,
+          :slug        => @slug,
+          :uid         => @uid,
+          :url         => @url,
+          :categories  => @categories,
+          :tags        => @tags,
+          :updates     => @updates,
+          :last_edit   => @last_edit,
+        }.compact
+      end
+    end
+  end
+end

data/lib/jekyll/embeddings-generator/version.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Jekyll
+  module EmbeddingsGenerator
+    VERSION = "0.1.0"
+  end
+end

data/lib/jekyll/processor.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+require_relative "commands/generator"
+require_relative "embeddings-generator/init"
+require_relative "embeddings-generator/version"
+require_relative "embeddings-generator/embeddings/generate"
+require_relative "embeddings-generator/embeddings/store"
+require_relative "embeddings-generator/models/data"
+require_relative "embeddings-generator/models/metadata"
+
+module Jekyll
+  module EmbeddingsGenerator
+    class Error < StandardError; end
+    class << self
+      attr_reader :config, :site
+
+      def run(options)
+        @config = Configuration.init(options)
+        @site = Configuration.build(options)
+        extract_content
+        write_related_posts
+      end
+
+      private
+
+      def extract_content
+        Jekyll.logger.info "Embeddings Generator:", "Starting to process markdown files..."
+        # Generate and store embeddings per each post
+        @site.posts.docs.each do |post|
+          Jekyll.logger.info "Embeddings Generator:", "Processing post: #{post.data["title"]}"
+          # Extract content and metadata
+          content = post.content
+          metadata = Jekyll::EmbeddingsGenerator::Metadata.new(post)
+
+          # Generate embeddings using OpenAI API
+          embedding = Jekyll::EmbeddingsGenerator::Embeddings.generate_embeddings(content)
+
+          # Store in Supabase
+          data = Jekyll::EmbeddingsGenerator::Data.new(post, embedding, metadata)
+          Jekyll::EmbeddingsGenerator::Store.store_embedding(data)
+        end
+        Jekyll.logger.info "Embeddings Generator:", "Finished processing markdown files."
+      end
+
+      def write_related_posts
+        # Query vector database and find related posts per each post
+        @site.posts.docs.each do |post|
+          # Find related posts
+          related_posts = Jekyll::EmbeddingsGenerator::Store.find_related(post)
+          write_to_file(related_posts, post)
+
+          # Log related posts for debugging
+          Jekyll.logger.info "Related posts:", "Found #{related_posts.length} related posts for #{post.data[@config["uid"]]}"
+        rescue StandardError => e
+          Jekyll.logger.error "Related posts:", "Error processing #{post.data["title"]}: #{e.message}"
+        end
+        Jekyll.logger.info "Related posts:", "Finished writing markdown files."
+      end
+
+      def write_to_file(data, post)
+        return if data.empty?
+
+        # Create directory if it doesn't exist
+        subdir = @config["path"]
+        target_dir = File.join(@site.source, "_data", subdir)
+        FileUtils.mkdir_p(target_dir)
+        # Write related posts to file, overwriting if exists
+        safe_uid = safe_filename(post.data[@config["uid"]].to_s)
+        filename = File.join(target_dir, "#{safe_uid}.yml")
+        File.write(filename, data.to_yaml, mode: "w")
+      end
+
+      def safe_filename(filename)
+        filename.downcase.gsub(%r![^a-z0-9\-_]!, "-")
+      end
+    end
+  end
+end

data/lib/jekyll-ai-related.rb

@@ -0,0 +1,3 @@
+require "jekyll"
+
+require_relative "jekyll/processor"

data/sig/jekyll/embeddings/generator.rbs

@@ -0,0 +1,6 @@
+module Jekyll
+  module EmbeddingsGenerator
+    VERSION: String
+    # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+  end
+end

data/sql/supabase/create.sql

@@ -0,0 +1,38 @@
+create extension if not exists vector;
+
+-- Create a table to store the embeddings
+--
+-- Notes: timezone must match those set in _config.yml in your Jekyll website.
+create table page_embeddings (
+  id bigint generated by default as identity primary key,
+  uid varchar(255) not null unique,
+  most_recent_edit timestamp with time zone default timezone('Europe/Rome'::text, now()) not null,
+  content text,
+  embedding vector(1536),
+  metadata jsonb,
+  created_at timestamp with time zone default timezone('Europe/Rome'::text, now())
+);
+
+-- Create a similarity search index
+create index page_embeddings_embedding_idx
+on page_embeddings
+using ivfflat (embedding vector_cosine_ops)
+with (lists = 100);
+
+-- create a Postgres function to safely execute the similarity search
+create or replace function related_posts(query text)
+returns table (
+  title text,
+  uid varchar(255),
+  most_recent_edit timestamp with time zone,
+  url text,
+  date text,
+  similarity float
+)
+language plpgsql
+security definer
+as $$
+begin
+  return query execute query;
+end;
+$$;

data/sql/supabase/drop.sql

@@ -0,0 +1,5 @@
+drop index if exists page_embeddings_embedding_idx;
+
+drop function if exists related_posts;
+
+drop table page_embeddings;