firehose-rb 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: ef4dac25083ab9471245c47df41233e04e19e12282153e43e630603195d7e1cc
+  data.tar.gz: 36db7c5b6534a76cbdd88158792fc614524ce7fa3ccce0ed396eb608fa478484
+SHA512:
+  metadata.gz: 9335a0468cb38b470b67edde432d05e6b96ac3e6209c2b66a51667754f9888eeccb9f8bf8944dcb91c175f15dbe8528343f01bc3c2e93429db55ccbb5003a2ac
+  data.tar.gz: cec5635b29ca8b3a592fc79b4333baf6e1ae472f47bf2858d0cd11e17849ebe5b55233cae584f87fae1039b8a6c8d282dd7561b8a66cfb5d962a8fcb8a974dcc

data/README.md

@@ -0,0 +1,79 @@
+# firehose-rb
+
+Ruby client for the Firehose real-time web monitoring API. SSE streaming with auto-reconnect, rules CRUD, and offset tracking.
+
+## Installation
+
+```ruby
+gem "firehose-rb", path: "../firehose-rb"
+```
+
+## Configuration
+
+```ruby
+Firehose.configure do |c|
+  c.management_key = "fhm_..."
+  c.tap_token = "fh_..."
+  c.base_url = "https://api.firehose.dev"  # default
+  c.timeout = 300                           # SSE timeout in seconds
+end
+```
+
+## Usage
+
+### Rules CRUD
+
+```ruby
+client = Firehose.client
+
+# Create a rule
+rule = client.create_rule(
+  value: '"AI agent" AND language:"en" AND recent:7d',
+  tag: "ai-agent",
+  quality: true
+)
+
+# List rules
+rules = client.list_rules
+
+# Delete a rule
+client.delete_rule(rule.id)
+```
+
+### Streaming
+
+```ruby
+client = Firehose.client
+
+# Track offsets for resume
+client.on_offset { |offset| save_offset(offset) }
+
+# Stream events (auto-reconnects with exponential backoff)
+client.stream(since: "1h") do |event|
+  event.id              # String
+  event.document.url    # String
+  event.document.title  # String
+  event.document.markdown # String (full page content)
+  event.matched_rule    # String (tag)
+  event.matched_at      # Time
+end
+
+# Stop streaming
+client.stop_stream
+```
+
+## Data Structures
+
+- `Firehose::Rule` — id, value, tag, quality, nsfw
+- `Firehose::Event` — id, document, matched_rule, matched_at
+- `Firehose::Document` — url, title, markdown, categories, types, language, publish_time
+
+## Error Handling
+
+- `Firehose::AuthenticationError` — invalid API keys
+- `Firehose::RateLimitError` — rate limited
+- `Firehose::ConnectionError` — connection failures
+- `Firehose::TimeoutError` — request timeout
+
+SSE streaming auto-reconnects with exponential backoff (1s → 2s → 4s → max 30s).
+Authentication errors are not retried.

data/lib/firehose/client.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require "faraday"
+require "json"
+
+module Firehose
+  class Client
+    def initialize(config: Firehose.configuration)
+      @config = config
+      @stream = Stream.new(config: config)
+    end
+
+    # Rules CRUD
+
+    def create_rule(value:, tag: nil, quality: false, nsfw: false)
+      body = { value: value, tag: tag, quality: quality, nsfw: nsfw }.compact
+      response = management_connection.post("/rules", body.to_json)
+      handle_response(response)
+      Rule.from_hash(JSON.parse(response.body))
+    end
+
+    def list_rules
+      response = management_connection.get("/rules")
+      handle_response(response)
+      JSON.parse(response.body).map { |r| Rule.from_hash(r) }
+    end
+
+    def delete_rule(rule_id)
+      response = management_connection.delete("/rules/#{rule_id}")
+      handle_response(response)
+      true
+    end
+
+    # Streaming
+
+    def stream(since: nil, &block)
+      raise ArgumentError, "block required" unless block_given?
+
+      @stream.connect(since: since, &block)
+    end
+
+    def on_offset(&block)
+      @stream.on_offset(&block)
+    end
+
+    def stop_stream
+      @stream.stop
+    end
+
+    private
+
+    def management_connection
+      @management_connection ||= Faraday.new(url: @config.base_url) do |f|
+        f.headers["Authorization"] = "Bearer #{@config.management_key}"
+        f.headers["Content-Type"] = "application/json"
+        f.adapter Faraday.default_adapter
+      end
+    end
+
+    def handle_response(response)
+      case response.status
+      when 200..299 then nil
+      when 401, 403 then raise AuthenticationError, "Invalid management key"
+      when 429 then raise RateLimitError, "Rate limited"
+      else raise Error, "HTTP #{response.status}: #{response.body}"
+      end
+    end
+  end
+end

data/lib/firehose/document.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Firehose
+  Document = Data.define(:url, :title, :markdown, :categories, :types, :language, :publish_time) do
+    def initialize(url:, title: nil, markdown: nil, categories: [], types: [], language: nil, publish_time: nil)
+      super
+    end
+
+    def self.from_hash(hash)
+      new(
+        url: hash["url"] || hash[:url],
+        title: hash["title"] || hash[:title],
+        markdown: hash["markdown"] || hash[:markdown],
+        categories: Array(hash["categories"] || hash[:categories]),
+        types: Array(hash["types"] || hash[:types]),
+        language: hash["language"] || hash[:language],
+        publish_time: parse_time(hash["publish_time"] || hash[:publish_time])
+      )
+    end
+
+    def self.parse_time(value)
+      return nil if value.nil?
+      return value if value.is_a?(Time)
+
+      Time.parse(value.to_s)
+    rescue ArgumentError
+      nil
+    end
+  end
+end

data/lib/firehose/error.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Firehose
+  class Error < StandardError; end
+  class AuthenticationError < Error; end
+  class RateLimitError < Error; end
+  class ConnectionError < Error; end
+  class TimeoutError < Error; end
+end

data/lib/firehose/event.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Firehose
+  Event = Data.define(:id, :document, :matched_rule, :matched_at) do
+    def initialize(id:, document:, matched_rule: nil, matched_at: nil)
+      super
+    end
+
+    def self.from_sse(data, id: nil)
+      parsed = JSON.parse(data)
+
+      doc = Document.from_hash(parsed["document"] || parsed)
+      matched_at = parsed["matched_at"] ? Time.parse(parsed["matched_at"]) : nil
+
+      new(
+        id: id || parsed["id"],
+        document: doc,
+        matched_rule: parsed["matched_rule"] || parsed["tag"],
+        matched_at: matched_at
+      )
+    rescue JSON::ParserError => e
+      raise Firehose::Error, "Failed to parse SSE event: #{e.message}"
+    end
+  end
+end

data/lib/firehose/rule.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Firehose
+  Rule = Data.define(:id, :value, :tag, :quality, :nsfw) do
+    def initialize(id:, value:, tag: nil, quality: false, nsfw: false)
+      super
+    end
+
+    def self.from_hash(hash)
+      new(
+        id: hash["id"] || hash[:id],
+        value: hash["value"] || hash[:value],
+        tag: hash["tag"] || hash[:tag],
+        quality: hash["quality"] || hash[:quality] || false,
+        nsfw: hash["nsfw"] || hash[:nsfw] || false
+      )
+    end
+  end
+end

data/lib/firehose/stream.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "uri"
+
+module Firehose
+  class Stream
+    MAX_BACKOFF = 30
+    INITIAL_BACKOFF = 1
+
+    attr_reader :last_event_id
+
+    def initialize(config:)
+      @config = config
+      @last_event_id = nil
+      @on_offset = nil
+      @running = false
+    end
+
+    def on_offset(&block)
+      @on_offset = block
+    end
+
+    def stop
+      @running = false
+    end
+
+    def connect(since: nil, &block)
+      @running = true
+      backoff = INITIAL_BACKOFF
+
+      while @running
+        begin
+          stream_events(since: since, &block)
+          backoff = INITIAL_BACKOFF
+        rescue Errno::ECONNREFUSED, Errno::ECONNRESET, Errno::ETIMEDOUT,
+               Net::OpenTimeout, Net::ReadTimeout, IOError => e
+          break unless @running
+
+          sleep(backoff)
+          backoff = [backoff * 2, MAX_BACKOFF].min
+          since = nil # use last_event_id on reconnect
+        rescue Firehose::AuthenticationError
+          raise
+        rescue StandardError => e
+          break unless @running
+
+          sleep(backoff)
+          backoff = [backoff * 2, MAX_BACKOFF].min
+          since = nil
+        end
+      end
+    end
+
+    private
+
+    def stream_events(since: nil, &block)
+      uri = build_uri(since: since)
+      headers = build_headers
+
+      Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https",
+                      open_timeout: 10, read_timeout: @config.timeout) do |http|
+        request = Net::HTTP::Get.new(uri, headers)
+        buffer = +""
+
+        http.request(request) do |response|
+          handle_response_status(response)
+
+          response.read_body do |chunk|
+            break unless @running
+
+            buffer << chunk
+            process_buffer(buffer, &block)
+          end
+        end
+      end
+    end
+
+    def build_uri(since: nil)
+      uri = URI.join(@config.base_url, "/stream")
+      params = {}
+      params["since"] = since if since
+      uri.query = URI.encode_www_form(params) if params.any?
+      uri
+    end
+
+    def build_headers
+      headers = {
+        "Accept" => "text/event-stream",
+        "Authorization" => "Bearer #{@config.tap_token}",
+        "Cache-Control" => "no-cache"
+      }
+      headers["Last-Event-ID"] = @last_event_id if @last_event_id
+      headers
+    end
+
+    def handle_response_status(response)
+      case response.code.to_i
+      when 200 then nil
+      when 401, 403 then raise Firehose::AuthenticationError, "Invalid tap token"
+      when 429 then raise Firehose::RateLimitError, "Rate limited"
+      else raise Firehose::ConnectionError, "HTTP #{response.code}: #{response.message}"
+      end
+    end
+
+    def process_buffer(buffer, &block)
+      while (idx = buffer.index("\n\n"))
+        raw_event = buffer.slice!(0, idx + 2)
+        parse_sse_event(raw_event, &block)
+      end
+    end
+
+    def parse_sse_event(raw, &block)
+      id = nil
+      data_lines = []
+
+      raw.each_line do |line|
+        line = line.chomp
+        if line.start_with?("id:")
+          id = line.sub("id:", "").strip
+        elsif line.start_with?("data:")
+          data_lines << line.sub("data:", "").strip
+        end
+      end
+
+      return if data_lines.empty?
+
+      data = data_lines.join("\n")
+      event = Event.from_sse(data, id: id)
+
+      @last_event_id = id if id
+      @on_offset&.call(@last_event_id)
+
+      block.call(event)
+    end
+  end
+end

data/lib/firehose.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require_relative "firehose/error"
+require_relative "firehose/rule"
+require_relative "firehose/document"
+require_relative "firehose/event"
+require_relative "firehose/stream"
+require_relative "firehose/client"
+
+module Firehose
+  class Configuration
+    attr_accessor :management_key, :tap_token, :base_url, :timeout
+
+    def initialize
+      @base_url = "https://api.firehose.dev"
+      @timeout = 300
+    end
+  end
+
+  class << self
+    def configuration
+      @configuration ||= Configuration.new
+    end
+
+    def configure
+      yield(configuration)
+    end
+
+    def reset_configuration!
+      @configuration = Configuration.new
+    end
+
+    def client
+      Client.new
+    end
+  end
+end

metadata

@@ -0,0 +1,91 @@
+--- !ruby/object:Gem::Specification
+name: firehose-rb
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Nauman Tariq
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: faraday
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.12'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.12'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.18'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.18'
+description: SSE streaming client with rules CRUD, auto-reconnect, and offset tracking
+  for the Firehose API.
+email:
+- nauman@intellecta.co
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- lib/firehose.rb
+- lib/firehose/client.rb
+- lib/firehose/document.rb
+- lib/firehose/error.rb
+- lib/firehose/event.rb
+- lib/firehose/rule.rb
+- lib/firehose/stream.rb
+homepage: https://github.com/nauman/firehose-rb
+licenses:
+- MIT
+metadata: {}
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.1'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.9
+specification_version: 4
+summary: Ruby client for the Firehose real-time web monitoring API
+test_files: []