forest_admin_agent 1.10.0 → 1.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7e8e0238988884b3db56ac66a0478dc662ea5fed4377d50b2de685f414c455cb
-  data.tar.gz: 7986a7262febae19442a08d0916362edd97ff655d2fad7f688d43babc14e8af1
+  metadata.gz: d4ece5f90c8cf5ecc42aeac33df998c280e581af076edd9fde12f34860f10737
+  data.tar.gz: 7dff9b95291a1add4102376a9e609a1e97c1a010c1f5a2292921de2e09cbf31d
 SHA512:
-  metadata.gz: d1f931091e7e9e836208337f190bc0442e7543e55afd232ffc2c98331823f1f235fa71eeae574b13a907009756783fd3d17592fe7540f091a827cee9a2e44ad2
-  data.tar.gz: 69f51337fc2eb92f6eba5f0c0d1decd260f61bb2703975336cec22c857452bb87ecbc5bc1a914444e5ec38a9052430df35f579124c3ac7a55cef61fd21a0152a
+  metadata.gz: 19b5b40cfc2492918a284f6409a79be709aea4d245e3673f9b811543327540be68cc850c2483cd1773672df7df62960fe450d702b9cb5a6caebdee01ecc0f687
+  data.tar.gz: b4fbf11c6492f9d3e2d75ebf6ec0ce08a850ec06fdcf7596c9abbe2cc6929d87fa5b70721e452acbd3015b9ebdd90a107e63c6573aed2d479c5980dfd03d250f

data/lib/forest_admin_agent/routes/resources/csv.rb

@@ -32,20 +32,41 @@ module ForestAdminAgent
                 )
               ]
             ),
-            page: QueryStringParser.parse_export_pagination(Facades::Container.config_from_cache[:limit_export_size]),
             search: QueryStringParser.parse_search(@collection, args),
-            search_extended: QueryStringParser.parse_search_extended(args)
+            search_extended: QueryStringParser.parse_search_extended(args),
+            sort: QueryStringParser.parse_sort(@collection, args),
+            segment: QueryStringParser.parse_segment(@collection, args)
           )
           projection = QueryStringParser.parse_projection(@collection, args)
-          records = @collection.list(@caller, filter, projection)
-          filename = args[:params][:filename] || "#{args[:params]["collection_name"]}.csv"
+          filename = args[:params][:filename] || args[:params]['collection_name']
           filename += '.csv' unless /\.csv$/i.match?(filename)
+          header = args[:params][:header]
+
+          # Generate timestamp for filename
+          now = Time.now.strftime('%Y%m%d_%H%M%S')
+          filename_with_timestamp = filename.gsub('.csv', "_export_#{now}.csv")
+
+          # Return streaming enumerator instead of full CSV string
+          list_records = ->(batch_filter) { @collection.list(@caller, batch_filter, projection) }
 
           {
             content: {
-              export: CsvGenerator.generate(records, projection)
+              type: 'Stream',
+              enumerator: Utils::CsvGeneratorStream.stream(
+                header,
+                filter,
+                projection,
+                list_records,
+                Facades::Container.config_from_cache[:limit_export_size]
+              ),
+              headers: {
+                'Content-Type' => 'text/csv; charset=utf-8',
+                'Content-Disposition' => "attachment; filename=\"#{filename_with_timestamp}\"",
+                'Cache-Control' => 'no-cache',
+                'X-Accel-Buffering' => 'no'
+              }
             },
-            filename: filename
+            status: 200
           }
         end
       end

data/lib/forest_admin_agent/routes/resources/related/csv_related.rb

@@ -30,28 +30,51 @@ module ForestAdminAgent
                   @permissions.get_scope(@collection),
                   ForestAdminAgent::Utils::QueryStringParser.parse_condition_tree(@child_collection, args)
                 ]
-              ),
-              page: QueryStringParser.parse_export_pagination(Facades::Container.config_from_cache[:limit_export_size])
+              )
             )
             projection = ForestAdminAgent::Utils::QueryStringParser.parse_projection_with_pks(@child_collection, args)
+
+            # Get the parent record ID
             id = Utils::Id.unpack_id(@collection, args[:params]['id'], with_key: true)
-            records = Collection.list_relation(
-              @collection,
-              id,
-              args[:params]['relation_name'],
-              @caller,
-              filter,
-              projection
-            )
+            relation_name = args[:params]['relation_name']
+
+            # Generate timestamp for filename
+            now = Time.now.strftime('%Y%m%d_%H%M%S')
+            collection_name = args.dig(:params, 'collection_name')
+            header = args.dig(:params, 'header')
+            filename_with_timestamp = "#{collection_name}_#{relation_name}_export_#{now}.csv"
 
-            filename = args[:params][:filename] || "#{args[:params]["relation_name"]}.csv"
-            filename += '.csv' unless /\.csv$/i.match?(filename)
+            # Create a callable to fetch related records
+            list_records = lambda do |batch_filter|
+              ForestAdminDatasourceToolkit::Utils::Collection.list_relation(
+                @collection,
+                id,
+                relation_name,
+                @caller,
+                batch_filter,
+                projection
+              )
+            end
 
+            # For related exports, use list_relation to fetch records
             {
               content: {
-                export: CsvGenerator.generate(records, projection)
+                type: 'Stream',
+                enumerator: ForestAdminAgent::Utils::CsvGeneratorStream.stream(
+                  header,
+                  filter,
+                  projection,
+                  list_records,
+                  Facades::Container.config_from_cache[:limit_export_size]
+                ),
+                headers: {
+                  'Content-Type' => 'text/csv; charset=utf-8',
+                  'Content-Disposition' => "attachment; filename=\"#{filename_with_timestamp}\"",
+                  'Cache-Control' => 'no-cache',
+                  'X-Accel-Buffering' => 'no'
+                }
               },
-              filename: filename
+              status: 200
             }
           end
         end

data/lib/forest_admin_agent/utils/csv_generator_stream.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+require 'csv'
+
+module ForestAdminAgent
+  module Utils
+    class CsvGeneratorStream
+      CHUNK_SIZE = 1000
+
+      # @param header [Array<String>] CSV header fields
+      # @param filter [ForestAdminDatasourceToolkit::Components::Query::Filter] Query filter
+      # @param projection [ForestAdminDatasourceToolkit::Components::Query::Projection] Fields to include
+      # @param list_records [Proc] A callable that fetches records: ->(batch_filter) { ... }
+      # @param limit_export_size [Integer, nil] Maximum number of records to export
+      # @return [Enumerator] Lazy enumerator that yields CSV rows
+      def self.stream(header, filter, projection, list_records, limit_export_size = nil)
+        Enumerator.new do |yielder|
+          # Yield header row first (client receives immediately)
+          header_array = parse_header(header, projection)
+          yielder << "#{header_array.join(",")}\n"
+
+          offset = 0
+
+          loop do
+            # Fetch batch of records
+            batch_filter = filter.override(
+              page: ForestAdminDatasourceToolkit::Components::Query::Page.new(offset: offset, limit: CHUNK_SIZE)
+            )
+            records = list_records.call(batch_filter)
+
+            # Break if no more records
+            break if records.empty?
+
+            # Convert each record to CSV row and yield immediately
+            records.each do |record|
+              yielder << generate_row(record, projection)
+            end
+
+            # Update offset
+            offset += CHUNK_SIZE
+
+            # Check if we've reached the export limit
+            break if limit_export_size && offset >= limit_export_size
+
+            # Check if this was a partial batch (last batch)
+            break if records.length < CHUNK_SIZE
+
+            # Periodic garbage collection to prevent memory creep
+            GC.start(full_mark: false) if (offset % 10_000).zero?
+          end
+        rescue IOError, Errno::EPIPE => e
+          # Client disconnected - clean up gracefully
+          Facades::Container.logger&.log(
+            'Info',
+            "CSV export interrupted at offset #{offset}: #{e.message}"
+          )
+        end
+      end
+
+      # Parse header parameter into an array
+      # @param header [String, Array, nil] Header as JSON string, array, or nil
+      # @param projection [Projection] Field projection (fallback if header is invalid)
+      # @return [Array<String>] Header fields
+      def self.parse_header(header, projection)
+        case header
+        when Array
+          header
+        when String
+          return projection.to_a if header.empty?
+
+          JSON.parse(header)
+        else
+          projection.to_a
+        end
+      rescue JSON::ParserError
+        # Fallback to projection if JSON parsing fails
+        projection.to_a
+      end
+
+      # Generate CSV row from record data
+      # @param record [Hash] Record data
+      # @param projection [Projection] Field projection
+      # @return [String] CSV data line
+      def self.generate_row(record, projection)
+        values = projection.map do |field|
+          value = ForestAdminDatasourceToolkit::Utils::Record.field_value(record, field)
+          format_value(value)
+        end
+        CSV.generate_line(values)
+      end
+
+      # Format individual value for CSV output
+      # @param value [Object] Value to format
+      # @return [String] Formatted value
+      def self.format_value(value)
+        case value
+        when nil
+          ''
+        when Array, Hash
+          value.to_json
+        when Date, DateTime, Time
+          value.respond_to?(:iso8601) ? value.iso8601 : value.to_s
+        else
+          value.to_s
+        end
+      end
+
+      private_class_method :parse_header, :generate_row, :format_value
+    end
+  end
+end

data/lib/forest_admin_agent/utils/schema/schema_emitter.rb

@@ -6,7 +6,7 @@ module ForestAdminAgent
     module Schema
       class SchemaEmitter
         LIANA_NAME = "agent-ruby"
-        LIANA_VERSION = "1.10.0"
+        LIANA_VERSION = "1.11.0"
 
         def self.generate(datasource)
           datasource.collections

data/lib/forest_admin_agent/version.rb

@@ -1,3 +1,3 @@
 module ForestAdminAgent
-  VERSION = "1.10.0"
+  VERSION = "1.11.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: forest_admin_agent
 version: !ruby/object:Gem::Version
-  version: 1.10.0
+  version: 1.11.0
 platform: ruby
 authors:
 - Matthieu
@@ -281,6 +281,7 @@ files:
 - lib/forest_admin_agent/utils/context_variables.rb
 - lib/forest_admin_agent/utils/context_variables_injector.rb
 - lib/forest_admin_agent/utils/csv_generator.rb
+- lib/forest_admin_agent/utils/csv_generator_stream.rb
 - lib/forest_admin_agent/utils/error_messages.rb
 - lib/forest_admin_agent/utils/id.rb
 - lib/forest_admin_agent/utils/query_string_parser.rb