active_record_json_streamer 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 6a470f59222637d0befa471e4d36732d148f664291ec90ee74fd5e534594050e
+  data.tar.gz: 4005d797b64d48177f3297aeef94022248c9014c2de23d05a4b8f5a82f4a562d
+SHA512:
+  metadata.gz: 594f50a9a2a7696bd0804c94ade45d8eb95f6ea13c4c5eb8afd11a7bcd283b55776dbea1fb37cf71012cc0fd54041f52e684d341ea701e049fa8a24e48e4fd5b
+  data.tar.gz: '059750a10d543534c5126cb0da438e04c535538f7ab6bff370f8c1a0e11a01fb95c9610e691fb108dff93016c75e03f3b6d571150bbba475ecc2722361df7963'

data/LICENSE.txt

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Mistika Team
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR A PARTICULAR PURPOSE AND LIABLE FOR
+ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
+THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,64 @@
+# ActiveRecordJsonStreamer
+
+`ActiveRecordJsonStreamer` is a lightweight Ruby gem that enables high-performance, memory-efficient JSON array HTTP streaming for large `ActiveRecord` queries using **Keyset Pagination** and **Chunked Transfer Encoding**.
+
+## Features
+
+- **Constant $O(1)$ RAM usage**: Streams records directly from the database in batches without accumulating large objects in memory.
+- **Keyset Pagination**: Fast cursor-based pagination over composite keys `(created_at, id)` instead of slow `OFFSET` queries.
+- **Rails Integration**: Out-of-the-box support for `ActionController::Base` and `ActionController::API`.
+- **Flexible Field Mapping**: Pass a block to customize the serialized JSON payload per record.
+
+## Installation
+
+Add this line to your application's `Gemfile`:
+
+```ruby
+gem "active_record_json_streamer"
+```
+
+And then execute:
+
+```bash
+$ bundle install
+```
+
+## Usage
+
+### In Rails Controllers
+
+```ruby
+class Admin::ExportLogsController < ApplicationController
+  include ActiveRecordJsonStreamer::Controller
+
+  def export
+    logs = UserActivityLog.where(created_at: 30.days.ago..)
+
+    stream_json_export(logs, filename: "logs.json", batch_size: 1000) do |log|
+      {
+        id: log.id,
+        action: log.action,
+        user_id: log.user_id,
+        created_at: log.created_at.iso8601
+      }
+    end
+  end
+end
+```
+
+### Options
+
+| Parameter | Type | Default | Description |
+|---|---|---|---|
+| `relation` | `ActiveRecord::Relation` | *Required* | The query relation to stream. |
+| `filename` | `String` | *Required* | Attachment filename for Content-Disposition header. |
+| `batch_size` | `Integer` | `1000` | Number of records to load per database query batch. |
+| `limit` | `Integer` / `nil` | `50_000` | Maximum total records to stream (`nil` for unlimited). |
+| `cursor_column` | `Symbol` | `:created_at` | Primary sort/cursor column. |
+| `primary_key` | `Symbol` | `:id` | Secondary sort/cursor column (unique identifier). |
+| `order` | `Symbol` | `:desc` | Sort direction (`:desc` or `:asc`). |
+
+## License
+
+This gem is available as open source under the terms of the [MIT License](LICENSE.txt).
+# active_record_json_streamer

data/lib/active_record_json_streamer/controller.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module ActiveRecordJsonStreamer
+  module Controller
+    DEFAULT_LIMIT = 50_000
+
+    def stream_json_export(relation, filename:, batch_size: 1000, limit: DEFAULT_LIMIT, cursor_column: :created_at, primary_key: :id, order: :desc, &field_mapper)
+      response.headers["Content-Type"] = "application/json"
+      response.headers["Content-Disposition"] = %(attachment; filename="#{filename}")
+      response.headers["Cache-Control"] = "no-cache"
+      response.headers["Last-Modified"] = Time.now.httpdate
+
+      streamer = Streamer.new(
+        relation,
+        batch_size: batch_size,
+        limit: limit,
+        cursor_column: cursor_column,
+        primary_key: primary_key,
+        order: order,
+        &field_mapper
+      )
+
+      self.response_body = streamer.enumerator
+    end
+  end
+end

data/lib/active_record_json_streamer/railtie.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module ActiveRecordJsonStreamer
+  class Railtie < ::Rails::Railtie
+    initializer "active_record_json_streamer.controller" do
+      ActiveSupport.on_load(:action_controller_base) do
+        include ActiveRecordJsonStreamer::Controller
+      end
+      ActiveSupport.on_load(:action_controller_api) do
+        include ActiveRecordJsonStreamer::Controller
+      end
+    end
+  end
+end

data/lib/active_record_json_streamer/streamer.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+require "json"
+
+module ActiveRecordJsonStreamer
+  class Streamer
+    DEFAULT_BATCH_SIZE = 1000
+    DEFAULT_LIMIT = 50_000
+
+    attr_reader :relation, :batch_size, :limit, :cursor_column, :primary_key, :order, :field_mapper
+
+    def initialize(relation, batch_size: DEFAULT_BATCH_SIZE, limit: DEFAULT_LIMIT, cursor_column: :created_at, primary_key: :id, order: :desc, &field_mapper)
+      @relation = relation
+      @batch_size = batch_size
+      @limit = limit
+      @cursor_column = cursor_column
+      @primary_key = primary_key
+      @order = order.to_sym
+      @field_mapper = field_mapper
+    end
+
+    def enumerator
+      Enumerator.new do |yielder|
+        yielder << "["
+        first = true
+        emitted = 0
+
+        col = relation.connection.quote_column_name(cursor_column)
+        pk = relation.connection.quote_column_name(primary_key)
+        tbl = relation.connection.quote_table_name(relation.table_name)
+        col_ref = "#{tbl}.#{col}"
+        pk_ref = "#{tbl}.#{pk}"
+
+        current_relation = relation.order(cursor_column => order, primary_key => order)
+
+        loop do
+          remaining = limit - emitted if limit
+          break if remaining && remaining <= 0
+
+          fetch_size = remaining ? [ batch_size, remaining ].min : batch_size
+          batch = current_relation.limit(fetch_size).to_a
+          break if batch.empty?
+
+          batch.each do |record|
+            yielder << "," unless first
+            first = false
+            data = field_mapper ? field_mapper.call(record) : record.as_json
+            yielder << JSON.generate(data)
+            emitted += 1
+          end
+
+          break if batch.size < fetch_size
+
+          last = batch.last
+          last_cursor = last.public_send(cursor_column)
+          last_pk = last.public_send(primary_key)
+
+          if order == :desc
+            current_relation = relation.where(
+              "(#{col_ref} < :ca) OR (#{col_ref} = :ca AND #{pk_ref} < :pk)",
+              ca: last_cursor, pk: last_pk
+            ).order(cursor_column => :desc, primary_key => :desc)
+          else
+            current_relation = relation.where(
+              "(#{col_ref} > :ca) OR (#{col_ref} = :ca AND #{pk_ref} > :pk)",
+              ca: last_cursor, pk: last_pk
+            ).order(cursor_column => :asc, primary_key => :asc)
+          end
+        end
+
+        yielder << "]"
+      end
+    end
+  end
+end

data/lib/active_record_json_streamer/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module ActiveRecordJsonStreamer
+  VERSION = "0.1.0"
+end

data/lib/active_record_json_streamer.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require_relative "active_record_json_streamer/version"
+require_relative "active_record_json_streamer/streamer"
+require_relative "active_record_json_streamer/controller"
+require_relative "active_record_json_streamer/railtie" if defined?(Rails::Railtie)
+
+module ActiveRecordJsonStreamer
+end

metadata

@@ -0,0 +1,116 @@
+--- !ruby/object:Gem::Specification
+name: active_record_json_streamer
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Alex Abramov
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '7.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '7.0'
+- !ruby/object:Gem::Dependency
+  name: actionpack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '7.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '7.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.12'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.12'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Stream large ActiveRecord relations as JSON arrays over HTTP using chunked
+  response bodies and Keyset pagination without OOM.
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE.txt
+- README.md
+- lib/active_record_json_streamer.rb
+- lib/active_record_json_streamer/controller.rb
+- lib/active_record_json_streamer/railtie.rb
+- lib/active_record_json_streamer/streamer.rb
+- lib/active_record_json_streamer/version.rb
+licenses:
+- MIT
+metadata: {}
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.0.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 4.0.16
+specification_version: 4
+summary: High-performance memory-efficient JSON array streaming for ActiveRecord with
+  Keyset pagination
+test_files: []