gromit 0.1.0 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 00dc903e9068922d891b726629b7e5e3ce058c2f073766775f1879fe2c5f20f9
-  data.tar.gz: 04e16aedf5357dfee62a94090c60c292af3fa5beba3e98d6066f1c119dec4d09
+  metadata.gz: f81be5c3be11802b49cafcd1c3fe572acadd5cee48cb9fc71a74f631ab95b3d2
+  data.tar.gz: 351fecf234d88f33670d78652b4caa28ff181ec9dae06a1ea0273b8759237606
 SHA512:
-  metadata.gz: dad20891a96a6589f59feb912b40eb99fc57dfe8adac93e9477a40af4386b5d7a8acf796365dffa31410afd3fe7a3f8a0ec2df5252590ca409ca83a46c318e1e
-  data.tar.gz: a60f72ed8958c7627bce422c713c30351682ba15dcd620ba1fddb6a65af27cecd76a5367787eed8cc9d6b1933425ef723669687c43dd7dbe5a50023b0fc3dba2
+  metadata.gz: eb0d33c49e7f3ece41f7e9fc59c3ca47eecdc4bf7113d9e50edd9353cbe0a278fd6a863b5fe2ca0ffdaf9c24cb91f8605336d4b1aae0c4001a43ab1cdbc82720
+  data.tar.gz: 8e0bd27547c8230ae29fedb5bdb7c095d357044f1160f9ac4c2b27ae0dcedc5b1bd63357a206e6e7661b3bd5909b409bce08809481ae8fa266837711b0879f94

data/MIT-LICENSE

@@ -1,4 +1,4 @@
-Copyright 2023 David Giffin
+Copyright 2023 Release Technologies, Inc.
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

data/README.md

@@ -1,8 +1,40 @@
 # Gromit
-Short description and motivation.
+Your documentation search assitant
 
 ## Usage
-How to use my plugin.
+
+Make sure to provide your OpenAI key in a `.env` file
+```
+OPENAPI_ACCESS_TOKEN=your-openai-token
+```
+
+To mount the Gromit engine in Rails 7 add this line to your routes:
+```ruby
+Rails.application.routes.draw do
+  mount Gromit::Engine => "/"
+end
+```
+
+Gromit provides the following routes:
+```
+Routes for Gromit::Engine:
+healthcheck GET  /healthcheck(.:format) gromit/gromit#healthcheck {:format=>:json}
+     search POST /search(.:format)      gromit/gromit#search {:format=>:json}
+     upsert POST /upsert(.:format)      gromit/gromit#upsert {:format=>:json}
+```
+
+To index your documentation locally you can use `gromit-reindexer`. This will update `redis-stack-server` running on your machine.
+```bash
+bundle exec gromit-reindexer -s /path/to/your/docs
+```
+
+To remotely upsert your documentation you can use `gromit-uploader`.
+```bash
+BASE_URL=https://gromit-rails.example.com bundle exec gromit-uploader -s /path/to/your/docs
+```
+
+For a working example of a Rails 7 application using Gromit check out:
+https://github.com/releasehub-com/gromit-example
 
 ## Installation
 Add this line to your application's Gemfile:

data/app/controllers/gromit/gromit_controller.rb

@@ -1,5 +1,6 @@
 module Gromit
   class GromitController < ApplicationController
+    skip_before_action :verify_authenticity_token
 
     def healthcheck
       gromit = Gromit::Search.new

data/app/helpers/gromit/application_helper.rb

@@ -1,4 +1,4 @@
-module Vapey
+module Gromit
   module Rails
     module ApplicationHelper
     end

data/app/jobs/gromit/application_job.rb

@@ -1,4 +1,4 @@
-module Vapey
+module Gomit
   module Rails
     class ApplicationJob < ActiveJob::Base
     end

data/app/mailers/gromit/application_mailer.rb

@@ -1,4 +1,4 @@
-module Vapey
+module Gromit
   module Rails
     class ApplicationMailer < ActionMailer::Base
       default from: "from@example.com"

data/lib/gromit/markdown_parser.rb

@@ -1,6 +1,6 @@
 
 class Gromit::MarkdownParser
-  attr_reader :sections
+  attr_reader :sections, :file_path
 
   class << self 
 
@@ -58,7 +58,6 @@ class Gromit::MarkdownParser
       data = redis.get(section_id)
 
       if data.nil?
-
         # OpenAI recommends replacing newlines with spaces for best results (specific to embeddings)
         input = section.gsub(/\n/m, ' ')
         response = openai.embeddings(parameters: { input: input, model: "text-embedding-ada-002"})
@@ -121,7 +120,7 @@ class Gromit::MarkdownParser
 
   def parse_file
     current_section = []
-    File.read(@file_path).lines do |line|
+    File.read(file_path).lines do |line|
       # Check if the line is a header (starts with one or more '#' characters)
       if header?(line)
         # Save the previous section if it's not empty

data/lib/gromit/uploader.rb

@@ -25,6 +25,7 @@ class Gromit::Uploader
       end.parse!
 
       path = Pathname.new(options.fetch(:source_dir, ''))
+      
       unless path.exist?
         puts "Error: The source directory (-s or --source) doesn't exist or is not specified."
         exit 1
@@ -33,7 +34,7 @@ class Gromit::Uploader
       sections = Gromit::MarkdownParser.process(path.to_s)
       sections.each do |section|
         puts "uploading: #{section[:file]} section: #{section[:section_title]}"
-        Uploader::Partay.post('/upsert', { headers: {"Content-Type": "application/json"}, body: section.to_json })
+        Partay.post('/upsert', { headers: {"Content-Type" => "application/json"}, body: section.to_json })
       end
     end
   end

data/lib/gromit/version.rb

@@ -1,3 +1,3 @@
 module Gromit
-  VERSION = "0.1.0"
+  VERSION = "0.1.2"
 end

data/lib/gromit.rb

@@ -4,6 +4,20 @@ require "gromit/engine"
 module Gromit
   class Search
 
+    EMBEDDINGS_SCHEMA = {
+      id: "TAG",
+      page_id: "TAG",
+      section_id: "TAG",
+      file: "TEXT",
+      title: "TEXT",
+      content: "TEXT",
+      checksum: "TEXT",
+      token_count: "NUMERIC",
+      embedding: "VECTOR FLAT 6 DIM 1536 DISTANCE_METRIC COSINE TYPE FLOAT64",
+    }
+
+    EMBEDDINGS_PREAMBLE = "FT.CREATE index ON JSON PREFIX 1 item: SCHEMA "
+
     def find_by_embedding(embedding)
       results = redis.call([
         "FT.SEARCH", "index", "@embedding:[VECTOR_RANGE $r $BLOB]=>{$YIELD_DISTANCE_AS: my_scores}",
@@ -29,24 +43,12 @@ module Gromit
     end
 
     def create_index
-      schema = {
-        id: "TAG",
-        page_id: "TAG",
-        section_id: "TAG",
-        file: "TEXT",
-        title: "TEXT",
-        content: "TEXT",
-        checksum: "TEXT",
-        token_count: "NUMERIC",
-        embedding: "VECTOR FLAT 6 DIM 1536 DISTANCE_METRIC COSINE TYPE FLOAT64",
-      }
-      preamble = "FT.CREATE index ON JSON PREFIX 1 item: SCHEMA "
-      command = (preamble + schema.map{|name,type| "$.#{name} AS #{name} #{type}"}.join(" ")).split(" ")
+      command = (EMBEDDINGS_PREAMBLE + EMBEDDINGS_SCHEMA.map{|name,type| "$.#{name} AS #{name} #{type}"}.join(" ")).split(" ")
       redis.call(command)
     end
 
     def redis
-      @redis ||= Redis.new(host: ENV.fetch("REDIS_HOST") { "127.0.0.1" }, port: ENV.fetch("REDIS_PORT") { "6379" }.to_i)
+      @redis ||= Gromit::MarkdownParser.redis
     end
 
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gromit
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.2
 platform: ruby
 authors:
 - David Giffin
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-05-05 00:00:00.000000000 Z
+date: 2023-05-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dotenv