llamaparserb 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: c22d4883933d23de15c34dd65a2e851c7294ab934fcb43be1fd79cafc8c95515
+  data.tar.gz: 6cde8b62919e0a73ccc7e73f00231c5711552f7a7dad1a8e29cccbc8c55551d4
+SHA512:
+  metadata.gz: 54fee6b5080d020caf2f28f77645336fea1fdbcace4442ffe2f842bb3101e63f0df62102b45e3af6caaae51a1d0c83cff5278fc870f3e1b0fb81ee36e4c7e0fd
+  data.tar.gz: ac05eeeaaceb34c47490797d8966ac12a3fd6e03050b1a4e51fe77ed8ff813d6deaaf3a85ba32219a47b1e53dd9ea25e51e91e1d582aa4f3e1d6c64d2f99c97d

data/CHANGELOG.md

@@ -0,0 +1,14 @@
+# Changelog
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+## [0.1.0] - 2024-11-27
+### Added
+- Initial release
+
+[Unreleased]: https://github.com/heidar/llamaparserb/compare/v0.1.0...HEAD
+[0.1.0]: https://github.com/heidar/llamaparserb/releases/tag/v0.1.0

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2024 Heidar Bernhardsson
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,114 @@
+# Llamaparserb
+
+A Ruby client for the LlamaIndex Parsing API. This gem allows you to easily parse various document formats (PDF, DOCX, etc.) into text or markdown. Loosely based on the Python version.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```bash
+gem 'llamaparserb'
+```
+
+And then execute:
+
+```bash
+$ bundle install
+```
+
+Or install it yourself as:
+
+```bash
+$ gem install llamaparserb
+```
+
+## Usage
+
+### Basic Usage
+
+```ruby
+require 'llamaparserb'
+
+# Initialize client with API key
+client = Llamaparserb::Client.new(ENV['LLAMA_CLOUD_API_KEY'])
+
+# Parse a file to text (default)
+text = client.parse_file('path/to/document.pdf')
+
+# Parse a file to markdown
+client = Llamaparserb::Client.new(ENV['LLAMA_CLOUD_API_KEY'], result_type: "markdown")
+markdown = client.parse_file('path/to/document.pdf')
+```
+
+### Advanced Options
+
+```ruby
+client = Llamaparserb::Client.new(
+  ENV['LLAMA_CLOUD_API_KEY'],
+  {
+    result_type: "markdown",  # Output format: "text" or "markdown"
+    num_workers: 4,           # Number of workers for concurrent processing
+    check_interval: 1,        # How often to check job status (seconds)
+    max_timeout: 2000,        # Maximum time to wait for parsing (seconds)
+    verbose: true,            # Enable detailed logging
+    language: :en,            # Target language
+    parsing_instruction: "",  # Custom parsing instructions
+    premium_mode: false,      # Enable premium parsing features
+    split_by_page: true       # Split result by pages
+  }
+)
+```
+
+### Supported File Types
+
+The client supports a wide range of file formats including:
+- Documents: PDF, DOCX, DOC, RTF, TXT
+- Presentations: PPT, PPTX
+- Spreadsheets: XLS, XLSX, CSV
+- Images: JPG, PNG, TIFF
+- And many more
+
+See `SUPPORTED_FILE_TYPES` constant for the complete list.
+
+## Error Handling
+
+By default, the client will return `nil` and print an error message if something goes wrong. You can change this behavior with the `ignore_errors` option:
+
+```ruby
+# Raise errors instead of returning nil
+client = Llamaparserb::Client.new(api_key, ignore_errors: false)
+```
+
+## Logging
+
+By default, the client uses Ruby's standard Logger with output to STDOUT. You can configure logging in several ways:
+
+```ruby
+# Use default logger with debug level output
+client = Llamaparserb::Client.new(api_key, verbose: true)
+
+# Use default logger with info level (less output)
+client = Llamaparserb::Client.new(api_key, verbose: false)
+
+# Use custom logger
+custom_logger = Logger.new('llamaparse.log')
+custom_logger.level = Logger::INFO
+client = Llamaparserb::Client.new(api_key, logger: custom_logger)
+
+# Use Rails logger in a Rails app
+client = Llamaparserb::Client.new(api_key, logger: Rails.logger)
+```
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/heidar/llamaparserb.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/lib/llamaparserb/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Llamaparserb
+  VERSION = "0.1.0"
+end

data/lib/llamaparserb.rb

@@ -0,0 +1,242 @@
+# frozen_string_literal: true
+
+require_relative "llamaparserb/version"
+require "faraday"
+require "faraday/multipart"
+require "json"
+require "mime/types"
+require "uri"
+require "async"
+require "logger"
+
+module Llamaparserb
+  class Error < StandardError; end
+
+  class Client
+    DEFAULT_BASE_URL = "https://api.cloud.llamaindex.ai/api/parsing"
+    DEFAULT_SEPARATOR = "\n---\n"
+    VALID_STATUSES = ["SUCCESS", "COMPLETED"].freeze
+    SUPPORTED_FILE_TYPES = [
+      ".pdf", ".602", ".abw", ".cgm", ".cwk", ".doc", ".docx", ".docm", ".dot",
+      ".dotm", ".hwp", ".key", ".lwp", ".mw", ".mcw", ".pages", ".pbd", ".ppt",
+      ".pptm", ".pptx", ".pot", ".potm", ".potx", ".rtf", ".sda", ".sdd", ".sdp",
+      ".sdw", ".sgl", ".sti", ".sxi", ".sxw", ".stw", ".sxg", ".txt", ".uof",
+      ".uop", ".uot", ".vor", ".wpd", ".wps", ".xml", ".zabw", ".epub", ".jpg",
+      ".jpeg", ".png", ".gif", ".bmp", ".svg", ".tiff", ".webp", ".htm", ".html",
+      ".xlsx", ".xls", ".xlsm", ".xlsb", ".xlw", ".csv", ".dif", ".sylk", ".slk",
+      ".prn", ".numbers", ".et", ".ods", ".fods", ".uos1", ".uos2", ".dbf",
+      ".wk1", ".wk2", ".wk3", ".wk4", ".wks", ".123", ".wq1", ".wq2", ".wb1",
+      ".wb2", ".wb3", ".qpw", ".xlr", ".eth", ".tsv"
+    ].freeze
+
+    attr_reader :api_key, :base_url, :options, :logger
+
+    def initialize(api_key = nil, options = {})
+      @api_key = api_key || ENV["LLAMA_CLOUD_API_KEY"]
+      raise Error, "API key is required" unless @api_key
+
+      @base_url = options[:base_url] || ENV["LLAMA_CLOUD_BASE_URL"] || DEFAULT_BASE_URL
+      @options = default_options.merge(options)
+      @logger = options[:logger] || default_logger
+      @connection = build_connection
+    end
+
+    def parse_file(file_path)
+      job_id = create_job(file_path)
+      log "Started parsing file under job_id #{job_id}", :info
+
+      wait_for_completion(job_id)
+
+      result = get_result(job_id)
+      log "Successfully retrieved result", :info
+      result
+    rescue => e
+      handle_error(e, file_path)
+    end
+
+    private
+
+    def default_options
+      {
+        result_type: :text,
+        num_workers: 4,
+        check_interval: 1,
+        max_timeout: 2000,
+        verbose: true,
+        show_progress: true,
+        language: :en,
+        parsing_instruction: "",
+        skip_diagonal_text: false,
+        invalidate_cache: false,
+        do_not_cache: false,
+        fast_mode: false,
+        premium_mode: false,
+        continuous_mode: false,
+        do_not_unroll_columns: false,
+        page_separator: nil,
+        page_prefix: nil,
+        page_suffix: nil,
+        gpt4o_mode: false,
+        gpt4o_api_key: nil,
+        guess_xlsx_sheet_names: false,
+        bounding_box: nil,
+        target_pages: nil,
+        ignore_errors: true,
+        split_by_page: true
+      }
+    end
+
+    def default_logger
+      logger = Logger.new($stdout)
+      logger.level = @options[:verbose] ? Logger::DEBUG : Logger::INFO
+      logger.formatter = proc do |severity, datetime, progname, msg|
+        "#{msg}\n"
+      end
+      logger
+    end
+
+    def log(message, level = :debug)
+      return unless @options[:verbose]
+      case level
+      when :info
+        logger.info(message)
+      when :warn
+        logger.warn(message)
+      when :error
+        logger.error(message)
+      else
+        logger.debug(message)
+      end
+    end
+
+    def wait_for_completion(job_id)
+      start_time = Time.now
+
+      loop do
+        sleep(@options[:check_interval])
+        response = get_job_status(job_id)
+        log "Status: #{response["status"]}", :debug
+
+        check_timeout(start_time, job_id)
+        break if job_completed?(response)
+        handle_error_status(response, job_id)
+      end
+    end
+
+    def job_completed?(response)
+      VALID_STATUSES.include?(response["status"])
+    end
+
+    def check_timeout(start_time, job_id)
+      return unless Time.now - start_time > @options[:max_timeout]
+      raise Error, "Job #{job_id} timed out after #{@options[:max_timeout]} seconds"
+    end
+
+    def handle_error_status(response, job_id)
+      if response["status"] == "ERROR"
+        error_code = response["error_code"] || "No error code found"
+        error_message = response["error_message"] || "No error message found"
+        raise Error, "Job failed: #{error_code} - #{error_message}"
+      end
+
+      unless response["status"] == "PENDING"
+        raise Error, "Unexpected status: #{response["status"]}"
+      end
+    end
+
+    def handle_error(error, file_path)
+      if @options[:ignore_errors]
+        log "Error while parsing file '#{file_path}': #{error.message}", :error
+        nil
+      else
+        raise error
+      end
+    end
+
+    def build_connection
+      Faraday.new(url: base_url) do |f|
+        f.request :multipart
+        f.request :json
+        f.response :json
+        f.response :raise_error
+        f.adapter Faraday.default_adapter
+      end
+    end
+
+    def create_job(file_path)
+      validate_file_type!(file_path)
+
+      file = Faraday::Multipart::FilePart.new(
+        file_path,
+        detect_content_type(file_path)
+      )
+
+      response = @connection.post("upload") do |req|
+        req.headers["Authorization"] = "Bearer #{api_key}"
+        req.body = upload_params(file)
+      end
+
+      response.body["id"]
+    end
+
+    def upload_params(file)
+      {
+        file: file,
+        language: @options[:language].to_s,
+        parsing_instruction: @options[:parsing_instruction],
+        invalidate_cache: @options[:invalidate_cache],
+        skip_diagonal_text: @options[:skip_diagonal_text],
+        do_not_cache: @options[:do_not_cache],
+        fast_mode: @options[:fast_mode],
+        premium_mode: @options[:premium_mode],
+        continuous_mode: @options[:continuous_mode],
+        do_not_unroll_columns: @options[:do_not_unroll_columns],
+        gpt4o_mode: @options[:gpt4o_mode],
+        gpt4o_api_key: @options[:gpt4o_api_key],
+        from_ruby_package: true
+      }.compact
+    end
+
+    def get_job_status(job_id)
+      response = @connection.get("job/#{job_id}") do |req|
+        req.headers["Authorization"] = "Bearer #{api_key}"
+      end
+
+      response.body
+    end
+
+    def get_result(job_id)
+      result_type = @options[:result_type].to_s
+      response = @connection.get("job/#{job_id}/result/#{result_type}") do |req|
+        req.headers["Authorization"] = "Bearer #{api_key}"
+      end
+
+      log "Result type: #{result_type}", :info
+      log "Raw response body: #{response.body.inspect}", :info
+
+      extract_content(response.body, result_type)
+    end
+
+    def extract_content(body, result_type)
+      content = if body.is_a?(Hash)
+        body[result_type] || body["content"]
+      else
+        body
+      end
+
+      log "Warning: No content found in response", :warn if content.nil?
+      content
+    end
+
+    def detect_content_type(filename)
+      MIME::Types.type_for(filename).first&.content_type || "application/octet-stream"
+    end
+
+    def validate_file_type!(file_path)
+      extension = File.extname(file_path).downcase
+      unless SUPPORTED_FILE_TYPES.include?(extension)
+        raise Error, "Unsupported file type: #{extension}. Supported types: #{SUPPORTED_FILE_TYPES.join(", ")}"
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,108 @@
+--- !ruby/object:Gem::Specification
+name: llamaparserb
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Heidar Bernhardsson
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2024-11-27 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: faraday
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: faraday-multipart
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: mime-types
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: async
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+description: A llamaparse client for Ruby.
+email:
+- heidar@heidarb.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- LICENSE.txt
+- README.md
+- lib/llamaparserb.rb
+- lib/llamaparserb/version.rb
+homepage: https://github.com/heidar/llamaparserb
+licenses:
+- MIT
+metadata:
+  allowed_push_host: https://rubygems.org
+  homepage_uri: https://github.com/heidar/llamaparserb
+  source_code_uri: https://github.com/heidar/llamaparserb
+  changelog_uri: https://github.com/heidar/llamaparserb/blob/master/CHANGELOG.md
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.0.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.5.11
+signing_key:
+specification_version: 4
+summary: A llamaparse client for Ruby.
+test_files: []