uptriever 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: e53b1a43458a2039d72c38133e6d92e55c4c715c5645d51c6801c4c6c7812ee1
+  data.tar.gz: 69bfee013e194759a5e8db274cf5d5bf690a6cb558fade52e9e5951114b91463
+SHA512:
+  metadata.gz: 8038fc632bd0b7afd1e715259c1eb5c3d77c5a1f9d72b09517dca1f1afd9b1541d5dce398d1edd1dbe9f310b9577f371d2f87e11bda69af63fe09946691a76d2
+  data.tar.gz: 31b1eae2eae167e0ec1786474e8d771339f3fc055cc5aba4dc467c83d5cb8c9e0aa0d99212bb2e20c99e72ce22334ccdf1326c45ba30557ba68d084557f73774

data/CHANGELOG.md

@@ -0,0 +1,9 @@
+# Change log
+
+## master
+
+## 0.1.0 (2024-07-24)
+
+- Initial release. ([@palkan][])
+
+[@palkan]: https://github.com/palkan

data/LICENSE.txt

@@ -0,0 +1,23 @@
+Copyright (c) 2024 Vladimir Dementyev
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+

data/README.md

@@ -0,0 +1,98 @@
+[![Gem Version](https://badge.fury.io/rb/uptriever.svg)](https://rubygems.org/gems/uptriever)
+[![Build](https://github.com/palkan/uptriever/workflows/Build/badge.svg)](https://github.com/palkan/uptriever/actions)
+
+# Uptriever
+
+Uptriever is a CLI to upload documentation source file (HTML, Markdown) to [Trieve][] for search indexing.
+
+## Installation
+
+Install Uptreiver as a Ruby gem (Ruby 3.1+ is required):
+
+```sh
+gem install uptriever
+```
+
+## Usage
+
+Currently, Uptriever requires an index configuration file (`.trieve.yml`) to be present in the documentation root folder containing the list of files to index and their metadata. A minimal example of indexing everything looks as follows:
+
+```yml
+hostname: https://myproject.example/docs
+pages:
+  - "**/*.md"
+```
+
+The `hostname` field is used to generate the `link` property for chunks (see [Trieve API](https://docs.trieve.ai/api-reference/chunk/create-or-upsert-chunk-or-chunks)).
+
+The `pages` field contains the list of pages to index. It supports glob patterns.
+
+With config in place, you can run the `uptriever` executable to perform the indexing:
+
+```sh
+$ uptriever -d ./docs --api-key=<Trieve API key> --dataset=<Trieve dataset>
+
+Groups: |===========================|
+Chunks: |===========================|
+```
+
+## Full-featured example
+
+Why do we need a configuration file? To leverage Trieve features such as groups, tags, and weights. Here is a real-life example:
+
+```yml
+# Ignore patterns for globs in pages
+ignore:
+ - "**/*/Readme.md"
+hostname: https://docs.anycable.io
+# Prepend file paths with this prefix.
+# Useful when you store documentation in multiple sources.
+url_prefix: anycable-go/
+
+# Make sure the following chunk groups are created
+groups:
+  - name: PRO version
+    tracking_id: pro
+  - name: Server
+    tracking_id: server
+  - name: Client
+    tracking_id: client
+  - name: Go package
+    tracking_id: package
+
+# Default metadata for pages (can be overriden)
+defaults:
+  groups: ["server"]
+  tags: ["docs"]
+
+pages:
+  # You can use a dictionary to define source paths
+  # along with metadata
+  - source: "./apollo.md"
+    groups: ["pro", "server"]
+  - source: "./binary_formats.md"
+    groups: ["pro", "server", "client"]
+  - "./broadcasting.md"
+  - "./broker.md"
+  - "./health_checking.md"
+  - "./instrumentation.md"
+  - source: "./library.md"
+    groups: ["package"]
+  - "./pubsub.md"
+  - source: "./js/**/*.md"
+    groups: ["client"]
+```
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at [https://github.com/palkan/uptriever](https://github.com/palkan/uptriever).
+
+## Credits
+
+This gem is generated via [`newgem` template](https://github.com/palkan/newgem) by [@palkan](https://github.com/palkan).
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+
+[Trieve]: https://trieve.ai

data/bin/uptriever

@@ -0,0 +1,13 @@
+#!/usr/bin/env ruby
+
+require "uptriever/cli"
+
+begin
+  cli = Uptriever::CLI.new
+  cli.run(ARGV)
+rescue => e
+  raise e if $DEBUG
+  STDERR.puts e.message
+  STDERR.puts e.backtrace.join("\n")
+  exit 1
+end

data/lib/uptriever/chunker.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require "nokogiri"
+
+module Uptriever
+  # Splits HTML into smaller chunks by h2 headers
+  class Chunker
+    attr_reader :chunk
+
+    def initialize(chunk)
+      @chunk = chunk
+    end
+
+    def chunks
+      doc = Nokogiri::HTML(chunk.fetch(:chunk_html))
+      header = doc.at_css("h1")
+      return [chunk_dup] unless header
+
+      # Root chunks are usually less specific, so make them weigh less
+      root_chunk = chunk_dup.tap {
+        _1[:weight] = 1.5
+        _1[:metadata] = {title: doc.at_css("h1").inner_text}
+      }
+      doc.xpath("//body").children.each_with_object([root_chunk]) do |child, acc|
+        # Start new chunk
+        if child.name == "h2"
+          anchor = child.inner_text.downcase.gsub(/[^a-z0-9]/, "-")
+          acc << chunk_dup.tap {
+            _1.merge!(
+              link: "#{_1.fetch(:link)}?id=#{anchor}",
+              tracking_id: "#{_1.fetch(:tracking_id)}##{anchor}",
+              metadata: {title: child.inner_text}
+            )
+          }
+          next acc
+        end
+
+        acc.last[:chunk_html] << child.to_xhtml
+      end
+    end
+
+    private
+
+    def chunk_dup = chunk.dup.tap { _1[:chunk_html] = +"" }
+  end
+end

data/lib/uptriever/cli.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require "uptriever"
+require "ruby-progressbar"
+require "optparse"
+
+module Uptriever
+  class CLI
+    def run(args = [])
+      @docs_dir = File.join(Dir.pwd, "docs")
+      @api_key = ENV["TRIEVE_API_KEY"]
+      @dataset = ENV["TRIEVE_DATASET"]
+      @dry_run = false
+
+      # Add optparser to parse options: --dir, --api_key, --dataset, --dry-run
+      OptionParser.new do |opts|
+        opts.banner = "Usage: uptriever [options]"
+        opts.on("-d", "--dir DIR", "Directory with documents") do |dir|
+          @docs_dir = dir
+        end
+        opts.on("-k", "--api-key API_KEY", "Trieve API key") do |key|
+          @api_key = key
+        end
+        opts.on("-s", "--dataset DATASET", "Trieve dataset") do |dataset|
+          @dataset = dataset
+        end
+        opts.on("--dry-run", "Dry run mode") do
+          @dry_run = true
+        end
+      end.parse!(args)
+
+      config = Config.new(@docs_dir)
+      client = Client.new(@api_key, @dataset, dry_run: @dry_run)
+
+      groups = config.groups
+      if groups.any?
+        progressbar = ProgressBar.create(title: "Groups", total: groups.size)
+        groups.each do
+          client.push_group(_1)
+          progressbar.increment
+        end
+      end
+
+      chunks = config.documents.flat_map { Chunker.new(_1.to_chunk_json).chunks }
+      progressbar = ProgressBar.create(title: "Chunks", total: chunks.size)
+      chunks.each do
+        client.push_chunk(_1)
+        progressbar.increment
+      end
+    end
+  end
+end

data/lib/uptriever/client.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "json"
+
+module Uptriever
+  class Client
+    BASE_URL = "https://api.trieve.ai/api"
+
+    attr_reader :headers
+    private attr_reader :dry_run
+
+    def initialize(api_key, dataset, dry_run: false)
+      @dry_run = dry_run
+      @headers = {
+        "Authorization" => api_key,
+        "TR-Dataset" => dataset
+      }.freeze
+    end
+
+    def push_group(group, upsert: true)
+      group[:upsert_by_tracking_id] = upsert
+      perform_request("/chunk_group", group.to_json)
+    end
+
+    def push_chunk(chunk, upsert: true)
+      chunk[:upsert_by_tracking_id] = upsert
+      perform_request("/chunk", chunk.to_json)
+    end
+
+    private
+
+    def perform_request(path, data)
+      uri = URI.parse(BASE_URL + path)
+
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.use_ssl = true if uri.scheme == "https"
+
+      request = Net::HTTP::Post.new(
+        uri.request_uri,
+        headers.merge("Content-Type" => "application/json")
+      )
+      request.body = data
+
+      if dry_run
+        puts "[DRY RUN] Perform POST #{path}: #{data}"
+        return
+      end
+
+      response = http.request(request)
+
+      if response.code.to_i != 200
+        raise "Invalid response code: #{response.code} (#{response.body[100...]})"
+      end
+
+      JSON.parse(response.body)
+    end
+  end
+end

data/lib/uptriever/config.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+require "yaml"
+
+module Uptriever
+  class Config
+    def self.parse(path) = new(path).documents
+
+    attr_reader :config_path, :root_dir
+
+    def initialize(root_dir)
+      @root_dir = root_dir
+      @config_path = File.join(root_dir, ".trieve.yml")
+      raise ArgumentError, ".trieve.yml is missing in the #{root_dir}" unless File.file?(config_path)
+    end
+
+    def groups
+      config["groups"] || []
+    end
+
+    def documents
+      pages = unwrap_pages(config["pages"])
+
+      defaults = (config["defaults"] || {}).transform_keys(&:to_sym)
+
+      pages.filter_map do |page|
+        next if config["ignore"]&.any? { File.fnmatch?(_1, page["source"]) }
+
+        relative_link = page["source"].sub(root_dir, "").sub(/\.[^\.]+$/, "").then do
+          next _1 unless config["url_prefix"]
+          File.join(config["url_prefix"], _1)
+        end
+
+        link = page["link"] || File.join(config.fetch("hostname"), relative_link)
+        id = page["id"] || relative_link.sub(/^\//, "").gsub(/[\/-]/, "-")
+
+        Document.new(id, page["source"], link, **defaults.merge({groups: page["groups"], tags: page["tags"], weight: page["weight"]}.compact))
+      end
+    end
+
+    private
+
+    def config = @config ||= YAML.load_file(config_path)
+
+    def unwrap_pages(items)
+      items.flat_map do |item|
+        if item.is_a?(String)
+          Dir.glob(File.expand_path(File.join(root_dir, item))).map { {"source" => _1} }
+        else
+          Dir.glob(File.expand_path(File.join(root_dir, item.fetch("source")))).map do
+            new_item = item.dup
+            new_item["source"] = _1
+            new_item
+          end
+        end
+      end
+    end
+  end
+end

data/lib/uptriever/document.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require "redcarpet"
+
+module Uptriever
+  class Document
+    attr_reader :id, :path, :link, :tags, :groups, :weight
+
+    def initialize(id, path, link, tags: nil, groups: nil, weight: 1.0)
+      @id = id
+      @path = path
+      @link = link
+      @tags = tags
+      @groups = groups
+      @weight = weight
+    end
+
+    def to_html
+      case File.extname(path)
+      when ".md"
+        markdown = Redcarpet::Markdown.new(Redcarpet::Render::HTML, autolink: true, tables: true)
+        markdown.render(File.read(path))
+      when ".html"
+        File.read(path)
+      else
+        raise ArgumentError, "Unsupported file type: #{path}"
+      end
+    end
+
+    def to_chunk_json
+      {
+        chunk_html: to_html,
+        link:,
+        tracking_id: id,
+        weight:
+      }.tap do
+        _1.merge!(tag_set: tags) if tags
+        _1.merge!(group_tracking_ids: groups) if groups
+      end
+    end
+  end
+end

data/lib/uptriever/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Uptriever # :nodoc:
+  VERSION = "0.0.1"
+end

data/lib/uptriever.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+require "uptriever/version"
+
+module Uptriever
+  autoload :Chunker, "uptriever/chunker"
+  autoload :Client, "uptriever/client"
+  autoload :Config, "uptriever/config"
+  autoload :Document, "uptriever/document"
+end

metadata

@@ -0,0 +1,214 @@
+--- !ruby/object:Gem::Specification
+name: uptriever
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Vladimir Dementyev
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2024-07-24 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: redcarpet
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.6'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: yaml
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: ruby-progressbar
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: uri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: optparse
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.15'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.15'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '13.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.23'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.23'
+description: Upload documenbts to Trieve
+email:
+- Vladimir Dementyev
+executables:
+- uptriever
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- LICENSE.txt
+- README.md
+- bin/uptriever
+- lib/uptriever.rb
+- lib/uptriever/chunker.rb
+- lib/uptriever/cli.rb
+- lib/uptriever/client.rb
+- lib/uptriever/config.rb
+- lib/uptriever/document.rb
+- lib/uptriever/version.rb
+homepage: https://github.com/palkan/uptriever
+licenses:
+- MIT
+metadata:
+  bug_tracker_uri: https://github.com/palkan/uptriever/issues
+  changelog_uri: https://github.com/palkan/uptriever/blob/master/CHANGELOG.md
+  documentation_uri: https://github.com/palkan/uptriever
+  homepage_uri: https://github.com/palkan/uptriever
+  source_code_uri: https://github.com/palkan/uptriever
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.1'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.4.20
+signing_key:
+specification_version: 4
+summary: Upload documenbts to Trieve
+test_files: []