gromit 0.1.1 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2bb63635be5d413e6221b47e7f726df939363b49601d67b5fde8657520b3c7b6
-  data.tar.gz: '009cb195bd61e73709dc26e4d11abcecc0fccc3ec774414bfab21426ba3522a5'
+  metadata.gz: 9932004042ffb0cc538889e78933675898b22305fa251e56cff964d3d79e46b0
+  data.tar.gz: 021d95202589af85ac778f35be87004ab36fafcd4c636aaedda0c0d9096f54e6
 SHA512:
-  metadata.gz: 79e8650c8531ab4bfa31ddca7af92c69d3dc160d6f4b66bf2f0c900f6a7ea451c21a00f64adb1394b813de92be33b7a7b28df7c40adf7864cc12543f66da049c
-  data.tar.gz: deb314cec627935551926ad76718c7c49623423c2d8f08a4b03ea38d1dade24750ee86f9592df3dc249c0e9639706efe104ab6d5336b3bd9c5e56123c5c25c91
+  metadata.gz: 7f2c28d1f9549c9271124eae5386b5e8b2b4aa79b03c31786f071c09471ecef517c0ac33301d5590ce03aac3eb6e0b2f615f1af50f28b646c7f8ff1807c4fe33
+  data.tar.gz: 716276f9c72f67a31d451f7ddbf4344dfd1a7d05d3e4ebea55e0c65da6b1ec018cbb53e163c1845ec7ed8433c48a0f5221144320578a47b1f5e7d41aa18b63b0

data/README.md

@@ -1,8 +1,40 @@
 # Gromit
-Short description and motivation.
+Your documentation search assitant
 
 ## Usage
-How to use my plugin.
+
+Make sure to provide your OpenAI key in a `.env` file
+```
+OPENAPI_ACCESS_TOKEN=your-openai-token
+```
+
+To mount the Gromit engine in Rails 7 add this line to your routes:
+```ruby
+Rails.application.routes.draw do
+  mount Gromit::Engine => "/"
+end
+```
+
+Gromit provides the following routes:
+```
+Routes for Gromit::Engine:
+healthcheck GET  /healthcheck(.:format) gromit/gromit#healthcheck {:format=>:json}
+     search POST /search(.:format)      gromit/gromit#search {:format=>:json}
+     upsert POST /upsert(.:format)      gromit/gromit#upsert {:format=>:json}
+```
+
+To index your documentation locally you can use `gromit-reindexer`. This will update `redis-stack-server` running on your machine.
+```bash
+bundle exec gromit-reindexer -s /path/to/your/docs
+```
+
+To remotely upsert your documentation you can use `gromit-uploader`.
+```bash
+BASE_URL=https://gromit-rails.example.com bundle exec gromit-uploader -s /path/to/your/docs
+```
+
+For a working example of a Rails 7 application using Gromit check out:
+https://github.com/releasehub-com/gromit-example
 
 ## Installation
 Add this line to your application's Gemfile:

data/app/controllers/gromit/gromit_controller.rb

@@ -1,5 +1,6 @@
 module Gromit
   class GromitController < ApplicationController
+    skip_before_action :verify_authenticity_token
 
     def healthcheck
       gromit = Gromit::Search.new

data/lib/gromit/markdown_parser.rb

@@ -1,6 +1,6 @@
 
 class Gromit::MarkdownParser
-  attr_reader :sections
+  attr_reader :sections, :file_path
 
   class << self 
 
@@ -58,7 +58,6 @@ class Gromit::MarkdownParser
       data = redis.get(section_id)
 
       if data.nil?
-
         # OpenAI recommends replacing newlines with spaces for best results (specific to embeddings)
         input = section.gsub(/\n/m, ' ')
         response = openai.embeddings(parameters: { input: input, model: "text-embedding-ada-002"})
@@ -121,7 +120,7 @@ class Gromit::MarkdownParser
 
   def parse_file
     current_section = []
-    File.read(@file_path).lines do |line|
+    File.read(file_path).lines do |line|
       # Check if the line is a header (starts with one or more '#' characters)
       if header?(line)
         # Save the previous section if it's not empty

data/lib/gromit/uploader.rb

@@ -25,6 +25,7 @@ class Gromit::Uploader
       end.parse!
 
       path = Pathname.new(options.fetch(:source_dir, ''))
+      
       unless path.exist?
         puts "Error: The source directory (-s or --source) doesn't exist or is not specified."
         exit 1
@@ -33,7 +34,7 @@ class Gromit::Uploader
       sections = Gromit::MarkdownParser.process(path.to_s)
       sections.each do |section|
         puts "uploading: #{section[:file]} section: #{section[:section_title]}"
-        Uploader::Partay.post('/upsert', { headers: {"Content-Type": "application/json"}, body: section.to_json })
+        Partay.post('/upsert', { headers: {"Content-Type" => "application/json"}, body: section.to_json })
       end
     end
   end

data/lib/gromit/version.rb

@@ -1,3 +1,3 @@
 module Gromit
-  VERSION = "0.1.1"
+  VERSION = "0.1.3"
 end

data/lib/gromit.rb

@@ -1,9 +1,24 @@
 require "gromit/version"
 require "gromit/engine"
+require "gromit/markdown_parser"
 
 module Gromit
   class Search
 
+    EMBEDDINGS_SCHEMA = {
+      id: "TAG",
+      page_id: "TAG",
+      section_id: "TAG",
+      file: "TEXT",
+      title: "TEXT",
+      content: "TEXT",
+      checksum: "TEXT",
+      token_count: "NUMERIC",
+      embedding: "VECTOR FLAT 6 DIM 1536 DISTANCE_METRIC COSINE TYPE FLOAT64",
+    }
+
+    EMBEDDINGS_PREAMBLE = "FT.CREATE index ON JSON PREFIX 1 item: SCHEMA "
+
     def find_by_embedding(embedding)
       results = redis.call([
         "FT.SEARCH", "index", "@embedding:[VECTOR_RANGE $r $BLOB]=>{$YIELD_DISTANCE_AS: my_scores}",
@@ -29,24 +44,12 @@ module Gromit
     end
 
     def create_index
-      schema = {
-        id: "TAG",
-        page_id: "TAG",
-        section_id: "TAG",
-        file: "TEXT",
-        title: "TEXT",
-        content: "TEXT",
-        checksum: "TEXT",
-        token_count: "NUMERIC",
-        embedding: "VECTOR FLAT 6 DIM 1536 DISTANCE_METRIC COSINE TYPE FLOAT64",
-      }
-      preamble = "FT.CREATE index ON JSON PREFIX 1 item: SCHEMA "
-      command = (preamble + schema.map{|name,type| "$.#{name} AS #{name} #{type}"}.join(" ")).split(" ")
+      command = (EMBEDDINGS_PREAMBLE + EMBEDDINGS_SCHEMA.map{|name,type| "$.#{name} AS #{name} #{type}"}.join(" ")).split(" ")
       redis.call(command)
     end
 
     def redis
-      @redis ||= Redis.new(host: ENV.fetch("REDIS_HOST") { "127.0.0.1" }, port: ENV.fetch("REDIS_PORT") { "6379" }.to_i)
+      @redis ||= Gromit::MarkdownParser.redis
     end
 
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gromit
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.3
 platform: ruby
 authors:
 - David Giffin
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-05-07 00:00:00.000000000 Z
+date: 2023-05-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dotenv