gromit 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 00dc903e9068922d891b726629b7e5e3ce058c2f073766775f1879fe2c5f20f9
+  data.tar.gz: 04e16aedf5357dfee62a94090c60c292af3fa5beba3e98d6066f1c119dec4d09
+SHA512:
+  metadata.gz: dad20891a96a6589f59feb912b40eb99fc57dfe8adac93e9477a40af4386b5d7a8acf796365dffa31410afd3fe7a3f8a0ec2df5252590ca409ca83a46c318e1e
+  data.tar.gz: a60f72ed8958c7627bce422c713c30351682ba15dcd620ba1fddb6a65af27cecd76a5367787eed8cc9d6b1933425ef723669687c43dd7dbe5a50023b0fc3dba2

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright 2023 David Giffin
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,28 @@
+# Gromit
+Short description and motivation.
+
+## Usage
+How to use my plugin.
+
+## Installation
+Add this line to your application's Gemfile:
+
+```ruby
+gem "gromit"
+```
+
+And then execute:
+```bash
+$ bundle
+```
+
+Or install it yourself as:
+```bash
+$ gem install gromit
+```
+
+## Contributing
+Contribution directions go here.
+
+## License
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,8 @@
+require "bundler/setup"
+
+APP_RAKEFILE = File.expand_path("test/dummy/Rakefile", __dir__)
+load "rails/tasks/engine.rake"
+
+load "rails/tasks/statistics.rake"
+
+require "bundler/gem_tasks"

data/app/assets/config/gromit_manifest.js

@@ -0,0 +1 @@
+//= link_directory ../stylesheets/gromit.css

data/app/assets/stylesheets/gromit/application.css

@@ -0,0 +1,15 @@
+/*
+ * This is a manifest file that'll be compiled into application.css, which will include all the files
+ * listed below.
+ *
+ * Any CSS and SCSS file within this directory, lib/assets/stylesheets, vendor/assets/stylesheets,
+ * or any plugin's vendor/assets/stylesheets directory can be referenced here using a relative path.
+ *
+ * You're free to add application-wide styles to this file and they'll appear at the bottom of the
+ * compiled file so the styles you add here take precedence over styles defined in any other CSS/SCSS
+ * files in this directory. Styles in this file should be added after the last require_* statement.
+ * It is generally better to create a new file per style scope.
+ *
+ *= require_tree .
+ *= require_self
+ */

data/app/controllers/gromit/application_controller.rb

@@ -0,0 +1,4 @@
+module Gromit
+  class ApplicationController < ActionController::Base
+  end
+end

data/app/controllers/gromit/gromit_controller.rb

@@ -0,0 +1,52 @@
+module Gromit
+  class GromitController < ApplicationController
+
+    def healthcheck
+      gromit = Gromit::Search.new
+
+      begin
+        # Send a PING command to Redis
+        redis_ping_response = gromit.redis.ping
+
+        # Check if the Redis server responded with "PONG"
+        if redis_ping_response == 'PONG'
+
+          # create the index if it's not already there
+          gromit.create_index
+        else
+          render json: { status: 'unhealthy', message: 'Redis connection is unhealthy' }
+        end
+      rescue Redis::CommandError, Redis::CannotConnectError => e
+        if e.message == "Index already exists"
+          render json: { status: 'healthy', message: 'Redis connection is healthy' }
+        else
+          render json: { status: 'unhealthy', message: "Redis connection error: #{e.message}" }
+        end
+      end
+    end
+
+    def search
+      gromit = Gromit::Search.new
+      result = gromit.find_by_embedding(params[:embedding])
+      render json: result
+    end
+
+    def upsert
+      gromit = Gromit::Search.new
+
+      # Hopefully don't have to do this
+      ## json_params = JSON.parse(request.raw_post) 
+
+      # Extract the key and value from the request data
+      id = params[:id]
+
+      # Upsert the record into the Redis database
+      gromit.redis.json_set("item:#{id}", Rejson::Path.root_path, params.deep_stringify_keys)
+
+      # Return a success response
+      render json: { status: 'success', message: "Record upserted successfully", key: "item:#{id}" }
+    end
+
+
+  end
+end

data/app/helpers/gromit/application_helper.rb

@@ -0,0 +1,6 @@
+module Vapey
+  module Rails
+    module ApplicationHelper
+    end
+  end
+end

data/app/jobs/gromit/application_job.rb

@@ -0,0 +1,6 @@
+module Vapey
+  module Rails
+    class ApplicationJob < ActiveJob::Base
+    end
+  end
+end

data/app/mailers/gromit/application_mailer.rb

@@ -0,0 +1,8 @@
+module Vapey
+  module Rails
+    class ApplicationMailer < ActionMailer::Base
+      default from: "from@example.com"
+      layout "mailer"
+    end
+  end
+end

data/app/models/gromit/application_record.rb

@@ -0,0 +1,5 @@
+module Gromit
+  class ApplicationRecord < ActiveRecord::Base
+    self.abstract_class = true
+  end
+end

data/app/views/layouts/gromit/application.html.erb

@@ -0,0 +1,15 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <title>Gromit</title>
+  <%= csrf_meta_tags %>
+  <%= csp_meta_tag %>
+
+  <%= stylesheet_link_tag    "gromit/application", media: "all" %>
+</head>
+<body>
+
+<%= yield %>
+
+</body>
+</html>

data/config/routes.rb

@@ -0,0 +1,8 @@
+Gromit::Engine.routes.draw do
+
+  scope '/', defaults: { format: :json } do
+    get '/healthcheck', to: 'gromit#healthcheck'
+    post '/search', to: 'gromit#search'
+    post '/upsert', to: 'gromit#upsert'
+  end
+end

data/lib/gromit/core.rb

@@ -0,0 +1,5 @@
+%w[
+  markdown_parser
+  reindexer
+  uploader
+].each { |name| require_relative name }

data/lib/gromit/engine.rb

@@ -0,0 +1,12 @@
+require "rails"
+
+module Gromit
+  class Engine < ::Rails::Engine
+    isolate_namespace Gromit
+
+    rake_tasks do
+      path = File.expand_path(__dir__)
+      Dir.glob("#{path}/tasks/**/*.rake").each { |f| load f }
+    end
+  end
+end

data/lib/gromit/markdown_parser.rb

@@ -0,0 +1,147 @@
+
+class Gromit::MarkdownParser
+  attr_reader :sections
+
+  class << self 
+
+    def redis
+      @@redis ||= Redis.new(host: ENV.fetch("REDIS_HOST") { "127.0.0.1" }, port: ENV.fetch("REDIS_PORT") { "6379" }.to_i)
+    end
+
+    def openai
+      @@openai ||= OpenAI::Client.new(access_token: ENV.fetch("OPENAPI_ACCESS_TOKEN"))
+    end
+
+    def process(directory)
+
+      git_file_list(directory).map do |file|
+        file_path = File.expand_path(File.join(directory, file))
+
+        next unless file.match(/\.md$/)
+
+        puts "processing file: #{file_path}"
+
+        page_id = Digest::SHA1.hexdigest(file_path)
+        title = File.read(file_path).lines.first.strip.gsub(/^#\s+/,'')
+        checksum = Digest::MD5.hexdigest(File.read(file_path))
+
+        parser = Gromit::MarkdownParser.new(file_path)
+        sections = parser.sections.map do |section|
+          section = process_markdown(section)
+          section_title = section.lines.first.strip.gsub(/^i[#]+\s+/,'')
+          section_id = Digest::MD5.hexdigest(section)
+          id = Digest::MD5.hexdigest("#{page_id}:#{section_id}")
+
+          # get cached embedding or call out to openai
+          token_count, embedding = get_embedding(section, section_id)
+
+          {
+            id: id,
+            page_id: page_id,
+            section_id: section_id,
+            file: file,
+            title: title,
+            section_title: section_title,
+            content: section,
+            checksum: checksum,
+            token_count: token_count,
+            embedding: embedding,
+          }
+
+        end
+      end.compact.flatten
+    end
+
+    def get_embedding(section, section_id)
+      token_count = nil
+      embedding = nil
+      data = redis.get(section_id)
+
+      if data.nil?
+
+        # OpenAI recommends replacing newlines with spaces for best results (specific to embeddings)
+        input = section.gsub(/\n/m, ' ')
+        response = openai.embeddings(parameters: { input: input, model: "text-embedding-ada-002"})
+
+        token_count = response['usage']['total_tokens']
+        embedding = response['data'].first['embedding']
+
+        redis.set(section_id, {'token_count' => token_count, 'embedding' => embedding}.to_json)
+      else
+        cached_embedding = JSON.parse(data)
+        token_count = cached_embedding['token_count']
+        embedding = cached_embedding['embedding']
+      end
+      [token_count, embedding]
+    end
+
+    def git_file_list(dir)
+      `cd #{dir} && git ls-files`.split("\n")
+    end
+
+    def process_markdown(file)
+      mkdocs_url = "https://docs-mkdocs.releaseapp.io"
+
+      # handle images
+      # TODO: deal with spaces??
+      file.gsub!(/(!\[[^\]]*?\])([\(<]+)[\.\/]+\.gitbook\/assets\/(.*?)([\)>]+)/m) do
+        "#{$1}#{$2}#{mkdocs_url}/img/#{$3}#{$4}"
+      end
+
+      # remove the .md extension from the end of the URLs from gitbook
+      file.gsub!(/(\[[^\]]+?\])\((.*?)\.md([#a-z0-9]*)\)/) do |match|
+        "#{$1}(#{$2}#{$3})" 
+      end
+
+      # handle "mentions" 
+      file.gsub!(/\[([^\]]+?).md\]\((.*?)\.md([#a-z0-9]*) "mention"\)/) do |match|
+        link = "#{$2}#{$3}"
+        title = $1.gsub("-", ' ').titleize
+        "[#{title}](#{link})" 
+      end
+
+      # convert gitbook hints to admonitions
+      # multi-line shortest match ...
+      file.gsub!(/{%\s+hint style="(.*?)"\s+?%}(.*?){% endhint %}/m) do |match|
+        ret = "!!! #{$1}\n"
+        ret += $2.lines.map{|line| "    #{line}" }.join()
+        ret
+      end
+
+      file
+    end
+
+  end
+
+  def initialize(file_path)
+    @file_path = file_path
+    @sections = []
+    parse_file
+  end
+
+  def parse_file
+    current_section = []
+    File.read(@file_path).lines do |line|
+      # Check if the line is a header (starts with one or more '#' characters)
+      if header?(line)
+        # Save the previous section if it's not empty
+        @sections << current_section.join unless current_section.empty?
+        # Start a new section
+        current_section = [line]
+      else
+        # Add the line to the current section
+        current_section << line
+      end
+    end
+    # Save the last section if it's not empty
+    @sections << current_section.join unless current_section.empty?
+  end
+
+  private
+
+  # Check if a line is a Markdown header
+  def header?(line)
+    line.strip.start_with?('#')
+  end
+end
+

data/lib/gromit/reindexer.rb

@@ -0,0 +1,72 @@
+require 'bundler/setup'
+require 'openai'
+require 'dotenv'
+require 'optparse'
+require 'redis'
+require 'rejson'
+require 'active_support/all'
+
+require_relative 'markdown_parser'
+
+Dotenv.load
+
+
+class Gromit::Reindexer
+  attr_accessor :redis
+
+  class << self
+    def invoke
+      options = { drop: false }
+      OptionParser.new do |opts|
+        opts.banner = "Usage: reindexer.rb [options]"
+
+        opts.on('-s', '--source SOURCE_DIR', 'Source directory') do |source_dir|
+          options[:source_dir] = source_dir
+        end
+
+        opts.on('-d', '--drop', 'Drop and create index before reindexing') do 
+          options[:drop] = true
+        end
+
+      end.parse!
+
+      # Validate the presence of source and destination directories
+      unless options[:source_dir]
+        puts "Error: source directory (-s or --source) must be specified."
+        exit 1
+      end
+
+      # Instantiate the ToMkDocs class and perform the conversion
+      reindexer = Gromit::Reindexer.new
+      reindexer.run(options[:source_dir], drop: options[:drop]) 
+
+      puts "Reindexer completed successfully."
+    end
+  end
+
+  def initialize
+    @redis ||= Redis.new(host: ENV.fetch("REDIS_HOST") { "127.0.0.1" }, port: ENV.fetch("REDIS_PORT") { "6379" }.to_i)
+  end
+
+  def run(directory = nil, drop: false)
+    gromit = Gromit::Search.new
+
+    if drop
+      gromit.recreate_index
+    end
+
+    directory ||= ENV.fetch("DOCS_DIRECTORY") { "/Users/david/development/docs/examples" }
+    sections = Gromit::MarkdownParser.process(directory)
+    sections.each do |section|
+      puts "indexing: #{section[:file]} section: #{section[:section_title]}"
+      data = section.stringify_keys
+      id = data['id']
+      gromit.redis.json_set("item:#{id}", Rejson::Path.root_path, data)
+    end
+  end
+
+end
+
+if __FILE__ == $0
+  Gromit::Reindexer::invoke
+end

data/lib/gromit/tasks/gromit.rake

@@ -0,0 +1,6 @@
+namespace :gromit do
+  desc "Reindex"
+  task :reindex do
+    # Task goes here
+  end
+end

data/lib/gromit/uploader.rb

@@ -0,0 +1,46 @@
+require 'bundler/setup'
+require 'openai'
+require 'httparty'
+require 'dotenv'
+require 'pathname'
+require 'optparse'
+
+require_relative 'markdown_parser'
+
+Dotenv.load
+
+
+class Gromit::Uploader
+
+  class << self
+    def invoke
+      options = {}
+
+      OptionParser.new do |opts|
+        opts.banner = "Usage: uploader.rb [options]"
+
+        opts.on('-s', '--source SOURCE_DIR', 'Source directory') do |source_dir|
+          options[:source_dir] = source_dir
+        end
+      end.parse!
+
+      path = Pathname.new(options.fetch(:source_dir, ''))
+      unless path.exist?
+        puts "Error: The source directory (-s or --source) doesn't exist or is not specified."
+        exit 1
+      end
+
+      sections = Gromit::MarkdownParser.process(path.to_s)
+      sections.each do |section|
+        puts "uploading: #{section[:file]} section: #{section[:section_title]}"
+        Uploader::Partay.post('/upsert', { headers: {"Content-Type": "application/json"}, body: section.to_json })
+      end
+    end
+  end
+
+  class Partay
+    include HTTParty
+    base_uri ENV.fetch("BASE_URL") { "127.0.0.1:9292" }
+  end
+
+end

data/lib/gromit/version.rb

@@ -0,0 +1,3 @@
+module Gromit
+  VERSION = "0.1.0"
+end

data/lib/gromit.rb

@@ -0,0 +1,53 @@
+require "gromit/version"
+require "gromit/engine"
+
+module Gromit
+  class Search
+
+    def find_by_embedding(embedding)
+      results = redis.call([
+        "FT.SEARCH", "index", "@embedding:[VECTOR_RANGE $r $BLOB]=>{$YIELD_DISTANCE_AS: my_scores}",
+        "PARAMS", "4", "BLOB", embedding.pack("E*"), "r", "5",
+        "LIMIT", "0", "10", "SORTBY", "my_scores", "DIALECT", "2"
+      ])
+      count = results[0]
+
+      output = []
+      results[1..].each_slice(2) do |key,value|
+        data = JSON.parse(value[3])
+        result = { key: key }.merge(data)
+        result.delete("embedding")
+        output << result
+      end
+
+      output
+    end
+
+    def recreate_index
+      redis.call(["FT.DROP", "index"])
+      create_index
+    end
+
+    def create_index
+      schema = {
+        id: "TAG",
+        page_id: "TAG",
+        section_id: "TAG",
+        file: "TEXT",
+        title: "TEXT",
+        content: "TEXT",
+        checksum: "TEXT",
+        token_count: "NUMERIC",
+        embedding: "VECTOR FLAT 6 DIM 1536 DISTANCE_METRIC COSINE TYPE FLOAT64",
+      }
+      preamble = "FT.CREATE index ON JSON PREFIX 1 item: SCHEMA "
+      command = (preamble + schema.map{|name,type| "$.#{name} AS #{name} #{type}"}.join(" ")).split(" ")
+      redis.call(command)
+    end
+
+    def redis
+      @redis ||= Redis.new(host: ENV.fetch("REDIS_HOST") { "127.0.0.1" }, port: ENV.fetch("REDIS_PORT") { "6379" }.to_i)
+    end
+
+  end
+end

data/lib/tasks/gromit_tasks.rake

@@ -0,0 +1,4 @@
+# desc "Explaining what the task does"
+# task :gromit do
+#   # Task goes here
+# end

metadata

@@ -0,0 +1,152 @@
+--- !ruby/object:Gem::Specification
+name: gromit
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- David Giffin
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2023-05-05 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: dotenv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: httparty
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 7.0.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 7.0.1
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.2'
+- !ruby/object:Gem::Dependency
+  name: rejson-rb
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.1
+- !ruby/object:Gem::Dependency
+  name: ruby-openai
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: gromit uses Redis and OpenAI embeddings to index your documentation
+email:
+- david@giffin.org
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- MIT-LICENSE
+- README.md
+- Rakefile
+- app/assets/config/gromit_manifest.js
+- app/assets/stylesheets/gromit/application.css
+- app/controllers/gromit/application_controller.rb
+- app/controllers/gromit/gromit_controller.rb
+- app/helpers/gromit/application_helper.rb
+- app/jobs/gromit/application_job.rb
+- app/mailers/gromit/application_mailer.rb
+- app/models/gromit/application_record.rb
+- app/views/layouts/gromit/application.html.erb
+- config/routes.rb
+- lib/gromit.rb
+- lib/gromit/core.rb
+- lib/gromit/engine.rb
+- lib/gromit/markdown_parser.rb
+- lib/gromit/reindexer.rb
+- lib/gromit/tasks/gromit.rake
+- lib/gromit/uploader.rb
+- lib/gromit/version.rb
+- lib/tasks/gromit_tasks.rake
+homepage: https://github.com/releasehub-com/gromit
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/releasehub-com/gromit
+  source_code_uri: https://github.com/releasehub-com/gromit
+  changelog_uri: https://github.com/releasehub-com/gromit
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.2.32
+signing_key:
+specification_version: 4
+summary: 'gromit: vector search and indexing of your documentation'
+test_files: []