url_to_markdown 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 2183cbb77e1eb9a58965d80e977b81698d81d5dc48ccb433d286e02dbcfa53e6
+  data.tar.gz: 30e5a87e074c00f456d67258b2ddbeed332dc6b2670e49915e356729560c0948
+SHA512:
+  metadata.gz: 66d0679ca3291058214204f6ac251898e7cc9fe348f3aa0ac5bf3167bb07d582180b49602191c6e023e62dfb8cc5395f8db6cdede0433fec289b926e46154e1c
+  data.tar.gz: 8e30c81de02f1360ab9e143aa21d947a0212c14ac1b0d05554e44c975a9b087d7083ebb13915390848b7530a3072eaf9f7b36747bfc6275f7703ae0a2da5f381

data/README.md

@@ -0,0 +1,206 @@
+# UrlToMarkdown
+
+Convert URLs (or raw HTML) into Markdown using a configurable processor.
+Cloudflare Browser Rendering is the default processor, but you can plug in your own.
+
+- Uses Cloudflare's rendering service for JavaScript-heavy pages by default.
+- Returns results as `SimpleResult::Success` / `SimpleResult::Failure`.
+- Supports optional caching via `UrlToMarkdown::CacheStore::PStore`.
+
+> Cloudflare Browser Rendering docs: https://workers.cloudflare.com/product/browser-rendering/
+
+## Requirements
+
+- Ruby 4.0.1
+- Cloudflare API token and account ID
+
+## Installation
+
+```bash
+bundle install
+```
+
+## Configuration
+
+```ruby
+require 'url_to_markdown'
+
+UrlToMarkdown.configure do |config|
+  config.cloudflare_api_token = ENV.fetch('CLOUDFLARE_API_TOKEN')
+  config.cloudflare_account_id = ENV.fetch('CLOUDFLARE_ACCOUNT_ID')
+  config.cloudflare_timeout_ms = 30_000
+  config.cloudflare_cache_ttl = 5
+  config.logger = Logger.new($stdout)
+end
+```
+
+## Basic Usage
+
+```ruby
+result = UrlToMarkdown.convert('https://example.com')
+
+if result.success?
+  puts result.payload
+else
+  warn result.error.message
+end
+```
+
+## Using a Cache Store
+
+`UrlToMarkdown` checks the cache before calling the processor. If `cache_store.exists?(url)` returns
+true, it returns the cached content from `cache_store.find_by(url)`. When the processor succeeds,
+it writes the markdown with `cache_store.store!(url, result.payload)`.
+
+```ruby
+cache = UrlToMarkdown::CacheStore::PStore.new(path: '/tmp/url_to_markdown.pstore')
+converter = UrlToMarkdown.new(url: 'https://example.com', cache_store: cache)
+
+result = converter.convert
+puts result.payload if result.success?
+```
+
+### Implementing Your Own Cache Store
+
+Implement the `UrlToMarkdown::CacheStore` interface with these methods:
+
+- `exists?(key)` → boolean
+- `find_by(key)` → cached value (raise `CacheReadError` if missing)
+- `store!(key, value)`
+- `invalidate!(key)`
+- `clear!`
+
+Example in-memory cache:
+
+```ruby
+class MemoryCache < UrlToMarkdown::CacheStore
+  def initialize
+    @store = {}
+  end
+
+  def exists?(key)
+    @store.key?(key)
+  end
+
+  def find_by(key)
+    @store.fetch(key) do
+      raise UrlToMarkdown::CacheReadError.new(nil, 'Cache miss')
+    end
+  end
+
+  def store!(key, value)
+    @store[key] = value
+  end
+
+  def invalidate!(key)
+    @store.delete(key)
+  end
+
+  def clear!
+    @store.clear
+  end
+end
+```
+
+## Custom Processor Options
+
+Use the Cloudflare processor directly for advanced options such as HTML input or dynamic rendering.
+
+```ruby
+processor = UrlToMarkdown::Cloudflare::Processor.new(
+  config: UrlToMarkdown.configuration,
+  logger: Logger.new($stdout)
+)
+
+result = processor.convert('https://example.com')
+```
+
+### Implementing Your Own Processor
+
+Custom processors must implement `convert(url)` and return a `SimpleResult::Success` or
+`SimpleResult::Failure`. You can also accept a `logger:` and `cache_store:` in `initialize` to match
+the base processor signature.
+
+```ruby
+class StaticProcessor < UrlToMarkdown::Processor
+  def initialize(logger: nil, cache_store: nil)
+    super
+  end
+
+  def convert(url)
+    markdown = "# Offline content for #{url}"
+    UrlToMarkdown::Result.success(markdown)
+  rescue StandardError => e
+    UrlToMarkdown::Result.failure(UrlToMarkdown::Error.new(e))
+  end
+end
+
+UrlToMarkdown.configure do |config|
+  config.default_processor = StaticProcessor
+end
+```
+
+### Rendering HTML Instead of a URL
+
+```ruby
+client = UrlToMarkdown::Cloudflare::Client.new(
+  token: ENV.fetch('CLOUDFLARE_API_TOKEN'),
+  account_id: ENV.fetch('CLOUDFLARE_ACCOUNT_ID')
+)
+
+result = client.markdown(html: '<h1>Hello</h1>')
+puts result.payload if result.success?
+```
+
+### Waiting for Dynamic Content
+
+```ruby
+client = UrlToMarkdown::Cloudflare::Client.new(
+  token: ENV.fetch('CLOUDFLARE_API_TOKEN'),
+  account_id: ENV.fetch('CLOUDFLARE_ACCOUNT_ID')
+)
+
+result = client.markdown(
+  url: 'https://spa-example.com',
+  wait_for_selector: '#main-content',
+  wait_for_timeout_in_milliseconds: 10_000
+)
+```
+
+## Error Handling
+
+Failures return a `SimpleResult::Failure` with a rich error type. Common errors include:
+
+- `UrlToMarkdown::MissingCredentialsError`
+- `UrlToMarkdown::AuthenticationError`
+- `UrlToMarkdown::RateLimitError`
+- `UrlToMarkdown::ServerError`
+- `UrlToMarkdown::InvalidUrlError`
+
+```ruby
+result = UrlToMarkdown.convert('https://example.com')
+
+result.on_error do |error|
+  warn "Conversion failed: #{error.class} - #{error.message}"
+end
+```
+
+## Development
+
+### Tests
+
+```bash
+bundle exec rake test
+```
+
+### Sorbet
+
+```bash
+bundle exec srb tc
+```
+
+### RuboCop
+
+```bash
+bundle exec rubocop
+```

data/lib/url_to_markdown/cache_store/pstore.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "pstore"
+
+class UrlToMarkdown
+  class CacheStore
+    class PStore < UrlToMarkdown::CacheStore
+      def initialize(path: "tmp/url_to_markdown.pstore")
+        @path = path
+        @store = ::PStore.new(path)
+      end
+
+      def exists?(key)
+        @store.transaction(true) do
+          !@store[key].nil?
+        end
+      rescue ::PStore::Error => e
+        raise UrlToMarkdown::CacheReadError, e
+      end
+
+      def find_by(key)
+        @store.transaction(true) do
+          value = @store[key]
+          raise UrlToMarkdown::CacheReadError.new(nil, "Cache miss") if value.nil?
+
+          value
+        end
+      rescue ::PStore::Error => e
+        raise UrlToMarkdown::CacheReadError, e
+      end
+
+      def store!(key, value)
+        @store.transaction do
+          @store[key] = value
+        end
+      rescue ::PStore::Error => e
+        raise UrlToMarkdown::CacheWriteError, e
+      end
+
+      def invalidate!(key)
+        @store.transaction do
+          @store.delete(key)
+        end
+      rescue ::PStore::Error => e
+        raise UrlToMarkdown::CacheWriteError, e
+      end
+
+      def clear!
+        FileUtils.rm_f(@path)
+      rescue StandardError => e
+        raise UrlToMarkdown::CacheWriteError, e
+      end
+    end
+  end
+end

data/lib/url_to_markdown/cache_store.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+class UrlToMarkdown
+  class CacheStore
+    def exists?(_key)
+      raise NotImplementedError, "Implement in subclass"
+    end
+
+    def find_by(_key)
+      raise NotImplementedError, "Implement in subclass"
+    end
+
+    def store!(_key, _value)
+      raise NotImplementedError, "Implement in subclass"
+    end
+
+    def invalidate!(_key)
+      raise NotImplementedError, "Implement in subclass"
+    end
+
+    def clear!
+      raise NotImplementedError, "Implement in subclass"
+    end
+  end
+end

data/lib/url_to_markdown/cloudflare/client.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+require "faraday"
+require "json"
+
+class UrlToMarkdown
+  module Cloudflare
+    class Client
+      BASE_URL = "https://api.cloudflare.com/client/v4"
+      DEFAULT_ACTION_TIMEOUT = 30_000
+
+      def initialize(token:, account_id:, action_timeout_in_milliseconds: nil)
+        @token = token
+        @account_id = account_id
+        @timeout = (action_timeout_in_milliseconds || DEFAULT_ACTION_TIMEOUT) / 1000.0
+
+        validate_credentials!
+      end
+
+      def markdown(url: nil, html: nil, wait_for_selector: nil, wait_for_timeout_in_milliseconds: nil, cache_ttl: nil)
+        validate_payload!(url: url, html: html)
+
+        response = connection.post("accounts/#{@account_id}/browser-rendering/markdown") do |request|
+          request.headers["Authorization"] = "Bearer #{@token}"
+          request.headers["Content-Type"] = "application/json"
+          request.options.timeout = @timeout
+          request.body = JSON.generate(build_payload(
+                                         url: url,
+                                         html: html,
+                                         wait_for_selector: wait_for_selector,
+                                         wait_for_timeout_in_milliseconds: wait_for_timeout_in_milliseconds,
+                                         cache_ttl: cache_ttl
+                                       ))
+        end
+
+        handle_response(response)
+      rescue Faraday::TimeoutError => e
+        UrlToMarkdown::Result.failure(UrlToMarkdown::TimeoutError.new(e))
+      rescue Faraday::ConnectionFailed => e
+        UrlToMarkdown::Result.failure(UrlToMarkdown::ConnectionError.new(e))
+      rescue Faraday::Error => e
+        UrlToMarkdown::Result.failure(UrlToMarkdown::NetworkError.new(e))
+      end
+
+      private
+
+      def validate_credentials!
+        return if @token && @account_id
+
+        raise UrlToMarkdown::MissingCredentialsError.new(nil, "Missing Cloudflare credentials")
+      end
+
+      def validate_payload!(url:, html:)
+        return if url || html
+
+        raise UrlToMarkdown::ValidationError.new(nil, "Provide a URL or HTML")
+      end
+
+      def build_payload(url:, html:, wait_for_selector:, wait_for_timeout_in_milliseconds:, cache_ttl:)
+        payload = {}
+        payload[:url] = url if url
+        payload[:html] = html if html
+        payload[:wait_for_selector] = wait_for_selector if wait_for_selector
+        if wait_for_timeout_in_milliseconds
+          payload[:wait_for_timeout_in_milliseconds] =
+            wait_for_timeout_in_milliseconds
+        end
+        payload[:cache_ttl] = cache_ttl if cache_ttl
+        payload
+      end
+
+      def connection
+        @connection ||= Faraday.new(url: BASE_URL)
+      end
+
+      def handle_response(response)
+        status = response.status
+        body = response.body.to_s
+
+        case status
+        when 200..299
+          data = JSON.parse(body)
+          if data.key?("result")
+            UrlToMarkdown::Result.success(data["result"])
+          else
+            UrlToMarkdown::Result.failure(UrlToMarkdown::MissingResultKeyInResponse.new(status, body))
+          end
+        when 401
+          UrlToMarkdown::Result.failure(UrlToMarkdown::AuthenticationError.new(status, body))
+        when 404
+          UrlToMarkdown::Result.failure(UrlToMarkdown::NotFoundError.new(status, body))
+        when 429
+          retry_after = response.headers["Retry-After"]
+          UrlToMarkdown::Result.failure(UrlToMarkdown::RateLimitError.new(status, body, retry_after: retry_after))
+        when 500..599
+          UrlToMarkdown::Result.failure(UrlToMarkdown::ServerError.new(status, body))
+        else
+          UrlToMarkdown::Result.failure(UrlToMarkdown::ApiError.new(status, body))
+        end
+      rescue JSON::ParserError
+        UrlToMarkdown::Result.failure(UrlToMarkdown::ApiError.new(status, body))
+      end
+    end
+  end
+end

data/lib/url_to_markdown/cloudflare/processor.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require "uri"
+
+class UrlToMarkdown
+  module Cloudflare
+    class Processor < UrlToMarkdown::Processor
+      def initialize(config: UrlToMarkdown.configuration, logger: nil, cache_store: nil)
+        super(logger: logger, cache_store: cache_store)
+        @config = config
+        @client = UrlToMarkdown::Cloudflare::Client.new(
+          token: @config.cloudflare_api_token!,
+          account_id: @config.cloudflare_account_id!,
+          action_timeout_in_milliseconds: @config.cloudflare_timeout_ms
+        )
+      end
+
+      def convert(url)
+        validate_url!(url)
+        @client.markdown(url: url)
+      end
+
+      private
+
+      def validate_url!(url)
+        uri = URI.parse(url)
+        return if uri.is_a?(URI::HTTP) && !uri.host.nil?
+
+        raise UrlToMarkdown::InvalidUrlError.new(nil, "Invalid URL")
+      rescue URI::InvalidURIError
+        raise UrlToMarkdown::InvalidUrlError.new(nil, "Invalid URL")
+      end
+    end
+  end
+end

data/lib/url_to_markdown/configuration.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require "logger"
+
+class UrlToMarkdown
+  class Configuration
+    attr_accessor :cloudflare_api_token, :cloudflare_account_id, :cloudflare_timeout_ms, :cloudflare_cache_ttl,
+                  :logger, :default_processor
+
+    def initialize
+      @cloudflare_api_token = ENV.fetch("CLOUDFLARE_API_TOKEN", nil)
+      @cloudflare_account_id = ENV.fetch("CLOUDFLARE_ACCOUNT_ID", nil)
+      @cloudflare_timeout_ms = 30_000
+      @cloudflare_cache_ttl = 5
+      @logger = Logger.new($stdout)
+      @default_processor = UrlToMarkdown::Cloudflare::Processor
+    end
+
+    def cloudflare_api_token!
+      return cloudflare_api_token if cloudflare_api_token && !cloudflare_api_token.empty?
+
+      raise UrlToMarkdown::MissingCredentialsError.new(nil, "Missing Cloudflare API token")
+    end
+
+    def cloudflare_account_id!
+      return cloudflare_account_id if cloudflare_account_id && !cloudflare_account_id.empty?
+
+      raise UrlToMarkdown::MissingCredentialsError.new(nil, "Missing Cloudflare account ID")
+    end
+  end
+end

data/lib/url_to_markdown/errors.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+class UrlToMarkdown
+  class Error < StandardError
+    attr_reader :original_error
+
+    def initialize(original_error = nil, message = nil)
+      @original_error = original_error
+      super(message || original_error&.message)
+    end
+  end
+
+  class ConfigurationError < Error; end
+  class MissingCredentialsError < ConfigurationError; end
+
+  class NetworkError < Error; end
+  class TimeoutError < NetworkError; end
+  class ConnectionError < NetworkError; end
+
+  class ApiError < Error
+    attr_reader :status_code, :response_body
+
+    def initialize(status_code = nil, response_body = nil, message: nil)
+      @status_code = status_code
+      @response_body = response_body
+      super(nil, message)
+    end
+  end
+
+  class AuthenticationError < ApiError; end
+
+  class RateLimitError < ApiError
+    attr_reader :retry_after
+
+    def initialize(status_code = nil, response_body = nil, retry_after: nil, message: nil)
+      @retry_after = retry_after
+      super(status_code, response_body, message: message)
+    end
+  end
+
+  class NotFoundError < ApiError; end
+  class ServerError < ApiError; end
+  class MissingResultKeyInResponse < ApiError; end
+
+  class ValidationError < Error; end
+  class InvalidUrlError < ValidationError; end
+
+  class CacheError < Error; end
+  class CacheReadError < CacheError; end
+  class CacheWriteError < CacheError; end
+end

data/lib/url_to_markdown/processor.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+class UrlToMarkdown
+  class Processor
+    def initialize(logger: nil, cache_store: nil)
+      @logger = logger
+      @cache_store = cache_store
+    end
+
+    def convert(_url)
+      raise NotImplementedError, "Implement in subclass"
+    end
+  end
+end

data/lib/url_to_markdown/result.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+class UrlToMarkdown
+  module Result
+    class << self
+      def success(value)
+        if SimpleResult.respond_to?(:success)
+          SimpleResult.success(value)
+        else
+          SimpleResult::Success.new(payload: value)
+        end
+      end
+
+      def failure(error)
+        if SimpleResult.respond_to?(:failure)
+          SimpleResult.failure(error)
+        else
+          SimpleResult::Failure.new(error: error)
+        end
+      end
+    end
+  end
+end

data/lib/url_to_markdown/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+class UrlToMarkdown
+  VERSION = "0.1.0"
+end

data/lib/url_to_markdown.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require "zeitwerk"
+require "simple_result"
+
+loader = Zeitwerk::Loader.for_gem
+loader.inflector.inflect("errors" => "Error", "pstore" => "PStore")
+loader.setup
+
+class UrlToMarkdown
+  def initialize(url:, processor: nil, logger: nil, cache_store: nil)
+    @url = url
+    @processor_class = processor || self.class.configuration.default_processor
+    @logger = logger || self.class.configuration.logger
+    @cache_store = cache_store
+  end
+
+  def convert
+    @logger.info("UrlToMarkdown: converting #{@url}")
+
+    if @cache_store&.exists?(@url)
+      cached = @cache_store.find_by(@url)
+      result = UrlToMarkdown::Result.success(cached)
+      @logger.info("UrlToMarkdown: cache hit for #{@url}")
+      @logger.info("UrlToMarkdown: completed #{@url}")
+      return result
+    end
+
+    processor = @processor_class.new(logger: @logger, cache_store: @cache_store)
+    result = processor.convert(@url)
+
+    @cache_store.store!(@url, result.payload) if @cache_store && result.respond_to?(:success?) && result.success?
+
+    @logger.info("UrlToMarkdown: completed #{@url}")
+    result
+  rescue StandardError => e
+    wrapped = e.is_a?(UrlToMarkdown::Error) ? e : UrlToMarkdown::Error.new(e)
+    UrlToMarkdown::Result.failure(wrapped)
+  end
+
+  class << self
+    def configuration
+      @configuration ||= UrlToMarkdown::Configuration.new
+    end
+
+    def configure
+      yield(configuration)
+    end
+
+    def convert(url, **)
+      new(url: url, **).convert
+    end
+  end
+end
+
+loader.eager_load

metadata

@@ -0,0 +1,109 @@
+--- !ruby/object:Gem::Specification
+name: url_to_markdown
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Lucian Ghinda
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: faraday
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: pstore
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+- !ruby/object:Gem::Dependency
+  name: simple-result
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+- !ruby/object:Gem::Dependency
+  name: zeitwerk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.6'
+email:
+- lucian@shortruby.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- lib/url_to_markdown.rb
+- lib/url_to_markdown/cache_store.rb
+- lib/url_to_markdown/cache_store/pstore.rb
+- lib/url_to_markdown/cloudflare/client.rb
+- lib/url_to_markdown/cloudflare/processor.rb
+- lib/url_to_markdown/configuration.rb
+- lib/url_to_markdown/errors.rb
+- lib/url_to_markdown/processor.rb
+- lib/url_to_markdown/result.rb
+- lib/url_to_markdown/version.rb
+homepage: https://github.com/lucianghinda/url_to_markdown
+licenses:
+- Apache-2.0
+metadata:
+  rubygems_mfa_required: 'true'
+  homepage_uri: https://github.com/lucianghinda/url_to_markdown
+  source_code_uri: https://github.com/lucianghinda/url_to_markdown
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.4.4
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 4.0.3
+specification_version: 4
+summary: Convert URLs to Markdown via Cloudflare Browser Rendering
+test_files: []