uptriever 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f44598b07e4b7c130e49c2308587dcc675eda01710f9ad24ce2fb4284fcfea63
-  data.tar.gz: 98ba686f341d7cdb66c06db19b2c0fba4c871a9c8b92c760177a6fd8f51d673e
+  metadata.gz: af61ca4ef13ebf666b7637be03e243fa891ba69052bb34bdd2424b612c6a5a42
+  data.tar.gz: f418f2b390df6e8bde1dfc34eafd2881174709272644c7b7a26f83f09d598e3f
 SHA512:
-  metadata.gz: 7627037babef9668fd456b79bd09e34686e543871e51eff72e8126ebf08113f3cc79517fa2a62846f192651e34d2e36bebb30a8d8d574aeeb9eef67685fa1f3a
-  data.tar.gz: a74448f80ab208acbf768c31464f730ea9402e73a68630fc4f71f372b93f26b573890d329640213dedc2afe977b9b9e5acd78fc59929e9285614ee16201677dc
+  metadata.gz: b07feade83badf77f8177e3f222780348427f33d3dd37f00d6935b32b97d0c253711fdb3c2c1a4112bef535b24a8fb925c500e0d4e684b884484a4f1a8c1f74c
+  data.tar.gz: ee8faddee9277a2c407b8b2f681b63fcc58a9a9f004a9642e6e1b890e3cd3dfc69a7f3c04498359e6b23b9f51fb4588818db2e28904c816b244d6368a1ea9362

data/CHANGELOG.md

@@ -2,6 +2,16 @@
 
 ## master
 
+## 0.2.0 (2025-08-05)
+
+- Ignore frontmatter. ([@palkan][])
+
+- Support `.mdx` ([@palkan][])
+
+## 0.1.2 (2025-01-03)
+
+- Fix the bug with links including PWD. ([@palkan][])
+
 ## 0.1.1 (2024-07-25)
 
 - Add ERB support to allow include dynamic fragments to configs. ([@palkan][])

data/lib/uptriever/chunker.rb

@@ -41,6 +41,6 @@ module Uptriever
 
     private
 
-    def chunk_dup = chunk.dup.tap { _1[:chunk_html] = +"" }
+    def chunk_dup = chunk.dup
   end
 end

data/lib/uptriever/client.rb

@@ -8,9 +8,10 @@ module Uptriever
     BASE_URL = "https://api.trieve.ai/api"
 
     attr_reader :headers
-    private attr_reader :dry_run
+    private attr_reader :dry_run, :dataset_id
 
-    def initialize(api_key, dataset, dry_run: false)
+    def initialize(api_key = ENV["TRIEVE_API_KEY"], dataset = ENV["TRIEVE_DATASET"], dry_run: false)
+      @dataset_id = dataset
       @dry_run = dry_run
       @headers = {
         "Authorization" => api_key,
@@ -25,22 +26,53 @@ module Uptriever
 
     def push_chunk(chunk, upsert: true)
       chunk[:upsert_by_tracking_id] = upsert
-      perform_request("/chunk", chunk.to_json)
+      perform_request("/chunk", chunk.to_json).inspect
+    end
+
+    def scroll_chunks(per_page: 100)
+      data = {
+        filters: {must: nil},
+        page_size: per_page
+      }
+
+      offset_id = nil
+
+      loop do
+        data[:offset_chunk_id] = offset_id if offset_id
+        data = perform_request("/chunks/scroll", data.to_json)
+
+        chunks = data.fetch("chunks")
+        chunks = chunks.select { _1["id"] != offset_id } if offset_id
+
+        break if chunks.empty?
+
+        chunks.each { yield _1 }
+
+        offset_id = chunks.last["id"]
+      end
+    end
+
+    def delete_chunk(id)
+      perform_request("/chunk/#{id}", method: :delete, expected_code: 204)
+    end
+
+    def usage
+      perform_request("/dataset/usage/#{dataset_id}", method: :get)
     end
 
     private
 
-    def perform_request(path, data)
+    def perform_request(path, data = nil, method: :post, expected_code: 200)
       uri = URI.parse(BASE_URL + path)
 
       http = Net::HTTP.new(uri.host, uri.port)
       http.use_ssl = true if uri.scheme == "https"
 
-      request = Net::HTTP::Post.new(
+      request = Net::HTTP.const_get(method.to_s.capitalize).new(
         uri.request_uri,
         headers.merge("Content-Type" => "application/json")
       )
-      request.body = data
+      request.body = data if data
 
       if dry_run
         puts "[DRY RUN] Perform POST #{path}: #{data}"
@@ -49,11 +81,11 @@ module Uptriever
 
       response = http.request(request)
 
-      if response.code.to_i != 200
+      if response.code.to_i != expected_code
         raise "Invalid response code: #{response.code} (#{response.body[100...]})"
       end
 
-      JSON.parse(response.body)
+      JSON.parse(response.body) if response.body
     end
   end
 end

data/lib/uptriever/config.rb

@@ -10,7 +10,7 @@ module Uptriever
     attr_reader :config_path, :root_dir
 
     def initialize(root_dir)
-      @root_dir = root_dir
+      @root_dir = File.expand_path(root_dir)
       @config_path = File.join(root_dir, ".trieve.yml")
       raise ArgumentError, ".trieve.yml is missing in the #{root_dir}" unless File.file?(config_path)
     end
@@ -32,7 +32,7 @@ module Uptriever
           File.join(config["url_prefix"], _1)
         end
 
-        link = page["link"] || File.join(config.fetch("hostname"), relative_link)
+        link = (page["link"] || File.join(config.fetch("hostname"), relative_link)).gsub(/([^:])\/{2,}/, '\1/')
         id = page["id"] || relative_link.sub(/^\//, "").gsub(/[\/-]/, "-")
 
         Document.new(id, page["source"], link, **defaults.merge({groups: page["groups"], tags: page["tags"], weight: page["weight"]}.compact))

data/lib/uptriever/document.rb

@@ -1,6 +1,9 @@
 # frozen_string_literal: true
 
 require "redcarpet"
+require "front_matter_parser"
+require "front_matter_parser/syntax_parser"
+FrontMatterParser::SyntaxParser::Mdx = FrontMatterParser::SyntaxParser::Md
 
 module Uptriever
   class Document
@@ -17,9 +20,10 @@ module Uptriever
 
     def to_html
       case File.extname(path)
-      when ".md"
+      when ".md", ".mdx"
+        parsed = FrontMatterParser::Parser.parse_file(path)
         markdown = Redcarpet::Markdown.new(Redcarpet::Render::HTML, autolink: true, tables: true)
-        markdown.render(File.read(path))
+        markdown.render(parsed.content)
       when ".html"
         File.read(path)
       else
@@ -29,7 +33,7 @@ module Uptriever
 
     def to_chunk_json
       {
-        chunk_html: to_html,
+        chunk_html: +to_html,
         link:,
         tracking_id: id,
         weight:

data/lib/uptriever/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Uptriever # :nodoc:
-  VERSION = "0.1.1"
+  VERSION = "0.2.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: uptriever
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - Vladimir Dementyev
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-07-25 00:00:00.000000000 Z
+date: 2025-08-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redcarpet
@@ -122,6 +122,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: front_matter_parser
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -206,7 +220,7 @@ metadata:
   documentation_uri: https://github.com/palkan/uptriever
   homepage_uri: https://github.com/palkan/uptriever
   source_code_uri: https://github.com/palkan/uptriever
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -222,7 +236,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.4.19
-signing_key: 
+signing_key:
 specification_version: 4
 summary: Upload documenbts to Trieve
 test_files: []