jekyll_ai_related_posts 0.1.2 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dde47b3847f5f2578a80f8f208049f912d409e7812a2927dda4ea333b0b8cd82
-  data.tar.gz: 9d94d4dd9695b96467349818d20f961ac9a55f9bd5c4968c3703636854d71edb
+  metadata.gz: bde6fb501500ac67e8b2fbcc05707dad8acc44d3e06ea9e461a98a065e9f84e2
+  data.tar.gz: e0a588ca8656583ad3dcbb36bb76d8c3992107ef807e5425c3f0f0f0db021da7
 SHA512:
-  metadata.gz: 857d8f3e381b1f4f3d52c411cc5ab630f4539ab735df57b33276f5b9df0647bebe44319b3a8a2a647f75cc3910409863aeae56d412dd48827fa5116870ea7975
-  data.tar.gz: eab136bfd7e226dcc0dfd71c14a0a9924b5e3aa59f8c376200129b3237e39f7e47ce540d55d03a080d834ebe56306b5827de44d8e55b9f04e71aedfbc3c9b60a
+  metadata.gz: a5e674a295aeb7c2e0bfd1bcec208d129dad7833a3690c4beb467e130220806e06867f49c4703a677cd000944ae0fdc66305bbcdc9679e7ccb51691f1ed9ec4b
+  data.tar.gz: f5c95c5eab1f65ec12b56b5ed949125eb24ec41a0e0097609bff313d654ff0888152e78070648fb744b4bf04c90a2f50a5698161d77df99545037d8c26f55d26

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 ## [Unreleased]
 
+## [0.1.3] - 2024-05-15
+
+- Better (nicer to read) log messages.
+
+## [0.1.2] - 2024-04-24
+
 - [Bugfix] Include up to 10 related posts (like native Jekyll does) instead of just 3.
 
 ## [0.1.1] - 2024-04-23

data/README.md

@@ -16,6 +16,13 @@ embeddings API that allows us to easily get the embedding vector (in one of
 OpenAI's models) of some text. We can use these vectors to compute related
 posts with the accuracy of OpenAI's models (or any other LLM, for that matter).
 
+### Used in Production at
+
+- [MikeKasberg.com](https://www.mikekasberg.com)
+
+(Feel free to open a PR to add your website if you're using this gem in
+production!)
+
 ## Installation
 
 Jekyll AI Related Posts is a [Jekyll

data/lib/jekyll_ai_related_posts/generator.rb

@@ -9,7 +9,13 @@ require "json"
 module JekyllAiRelatedPosts
   class Generator < Jekyll::Generator
     def generate(site)
+      Jekyll.logger.debug "AI Related Posts:", "Generating related posts..."
+
       @site = site
+      @stats = {
+        cache_hits: 0,
+        cache_misses: 0
+      }
       setup_database
 
       @indexed_posts = {}
@@ -18,7 +24,6 @@ module JekyllAiRelatedPosts
       end
 
       if fetch_enabled?
-        Jekyll.logger.info "[ai_related_posts] Generating related posts..."
         @embeddings_fetcher = new_fetcher
 
         @site.posts.docs.each do |p|
@@ -28,13 +33,25 @@ module JekyllAiRelatedPosts
         @site.posts.docs.each do |p|
           find_related(p)
         end
+        Jekyll.logger.info "AI Related Posts:", "Found #{@stats[:cache_hits]} cached embeddings; fetched #{@stats[:cache_misses]}"
       else
-        Jekyll.logger.info "[ai_related_posts] Using cached related posts data..."
+        Jekyll.logger.info "AI Related Posts:", "Fetch disabled. Using cached related posts data."
 
         @site.posts.docs.each do |p|
           fallback_generate_related(p)
         end
+
+        case @stats[:cache_misses]
+        when 0
+          Jekyll.logger.info "AI Related Posts:", "Found #{@stats[:cache_hits]} cached embeddings; all embeddings cached"
+        when 1
+          Jekyll.logger.info "AI Related Posts:", "Found #{@stats[:cache_hits]} cached embeddings; skipped 1 fetch"
+        else
+          Jekyll.logger.info "AI Related Posts:", "Found #{@stats[:cache_hits]} cached embeddings; skipped #{@stats[:cache_misses]} fetches"
+        end
       end
+
+      Jekyll.logger.debug "AI Related Posts:", "Done generating related posts"
     end
 
     private
@@ -53,8 +70,14 @@ module JekyllAiRelatedPosts
     def fallback_generate_related(post)
       existing = Models::Post.find_by(relative_path: post.relative_path)
       if existing.nil?
+        @stats[:cache_misses] += 1
         post.data["ai_related_posts"] = post.related_posts
       else
+        if existing.embedding_text == embedding_text(post)
+          @stats[:cache_hits] += 1
+        else
+          @stats[:cache_misses] += 1
+        end
         find_related(post)
       end
     end
@@ -80,20 +103,25 @@ module JekyllAiRelatedPosts
         existing = nil
       end
 
-      return unless existing.nil?
+      if existing.nil?
+        @stats[:cache_misses] += 1
 
-      Models::Post.create!(
-        relative_path: post.relative_path,
-        embedding_text: embedding_text(post),
-        embedding: embedding_for(post).to_json
-      )
+        Models::Post.create!(
+          relative_path: post.relative_path,
+          embedding_text: embedding_text(post),
+          embedding: embedding_for(post).to_json
+        )
 
-      sql = <<-SQL
-          INSERT INTO vss_posts (rowid, post_embedding)
-            SELECT rowid, embedding FROM posts WHERE relative_path = :relative_path;
-      SQL
-      ActiveRecord::Base.connection.execute(ActiveRecord::Base.sanitize_sql([ sql,
-                                                                             { relative_path: post.relative_path } ]))
+        sql = <<-SQL
+            INSERT INTO vss_posts (rowid, post_embedding)
+              SELECT rowid, embedding FROM posts WHERE relative_path = :relative_path;
+        SQL
+        ActiveRecord::Base.connection.execute(
+          ActiveRecord::Base.sanitize_sql([ sql, { relative_path: post.relative_path } ])
+        )
+      else
+        @stats[:cache_hits] += 1
+      end
     end
 
     def find_related(post)
@@ -107,9 +135,9 @@ module JekyllAiRelatedPosts
         LIMIT 10000;
       SQL
 
-      results = ActiveRecord::Base.connection.execute(ActiveRecord::Base.sanitize_sql([ sql, {
-                                                                                        relative_path: post.relative_path
-                                                                                      } ]))
+      results = ActiveRecord::Base.connection.execute(
+        ActiveRecord::Base.sanitize_sql([ sql, { relative_path: post.relative_path } ])
+      )
       # The first result is the post itself, with a distance of 0.
       rowids = results.sort_by { |r| r["distance"] }.drop(1).first(10).map { |r| r["rowid"] }
 
@@ -139,16 +167,23 @@ module JekyllAiRelatedPosts
     end
 
     def embedding_for(post)
-      Jekyll.logger.info "[ai_related_posts] Fetching embedding for #{post.relative_path}"
+      Jekyll.logger.info "AI Related Posts:", "Fetching embedding for #{post.relative_path}"
       input = embedding_text(post)
 
       @embeddings_fetcher.embedding_for(input)
     end
 
     def setup_database
+      db_path = @site.in_source_dir(".ai_related_posts_cache.sqlite3")
+      if File.exist?(db_path)
+        Jekyll.logger.debug "AI Related Posts:", "Found cache [.ai_related_posts_cache.sqlite3]"
+      else
+        Jekyll.logger.info "AI Related Posts:", "Creating cache [.ai_related_posts_cache.sqlite3]"
+      end
+
       ActiveRecord::Base.establish_connection(
         adapter: "sqlite3",
-        database: @site.in_source_dir(".ai_related_posts_cache.sqlite3")
+        database: db_path
       )
       # We don't need WAL mode for this.
       ActiveRecord::Base.connection.execute("PRAGMA journal_mode=DELETE;")
@@ -175,7 +210,7 @@ module JekyllAiRelatedPosts
       SQL
       ActiveRecord::Base.connection.execute(create_vss_posts)
 
-      Jekyll.logger.debug("ai_related_posts db setup complete")
+      Jekyll.logger.debug "AI Related Posts:", "DB setup complete"
     end
   end
 end

data/lib/jekyll_ai_related_posts/open_ai_embeddings.rb

@@ -29,8 +29,8 @@ module JekyllAiRelatedPosts
 
       res.body["data"].first["embedding"]
     rescue Faraday::Error => e
-      Jekyll.logger.error "Error response from OpanAI API!"
-      Jekyll.logger.error e.inspect
+      Jekyll.logger.error "AI Related Posts:", "Error response from OpenAI API!"
+      Jekyll.logger.error "AI Related Posts:", e.inspect
 
       raise
     end

data/lib/jekyll_ai_related_posts/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module JekyllAiRelatedPosts
-  VERSION = "0.1.2"
+  VERSION = "0.1.4"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: jekyll_ai_related_posts
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.4
 platform: ruby
 authors:
 - Mike Kasberg
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-04-24 00:00:00.000000000 Z
+date: 2024-10-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord