activeadmin_batched_export 0.2.0 → 0.3.0

data/app/assets/controllers/activeadmin_batched_export/batched_export_controller.js

@@ -1,8 +1,22 @@
 import { Controller } from "@hotwired/stimulus"
+import {
+  appendExportColumnParams,
+  assembleExportParts,
+} from "activeadmin_batched_export/chunk_assembly"
 
-// Progressive ActiveAdmin export: sequential batch fetches + single client-side save.
+// Progressive ActiveAdmin export: sequential cursor fetches + single client-side save.
 export default class extends Controller {
-  static targets = ["progressWrap", "status", "fraction", "bar", "error", "start", "save", "columnCheckbox"]
+  static targets = [
+    "progressWrap",
+    "status",
+    "fraction",
+    "bar",
+    "error",
+    "start",
+    "save",
+    "cancel",
+    "columnCheckbox",
+  ]
   static values = {
     metaUrl: String,
     meta: Object,
@@ -14,6 +28,9 @@ export default class extends Controller {
     failedBatchTemplate: String,
     readyMessage: String,
     needsColumnMessage: String,
+    cancelledMessage: String,
+    incompleteMessage: String,
+    overMaxMessage: String,
   }
 
   connect() {
@@ -21,6 +38,7 @@ export default class extends Controller {
     this.filename = null
     this.mime = "application/octet-stream"
     this.readyBlob = null
+    this.abortController = null
     this.hideError()
   }
 
@@ -44,18 +62,47 @@ export default class extends Controller {
 
   /** Appends export_columns[]=… from checked column checkboxes (all checked by default). */
   urlWithExportColumns(urlString) {
-    const u = new URL(urlString, window.location.origin)
-    u.searchParams.delete("export_columns[]")
-    u.searchParams.delete("export_columns")
-
-    if (!this.hasColumnCheckboxTarget) return u.toString()
+    if (!this.hasColumnCheckboxTarget) {
+      return appendExportColumnParams(urlString, [], window.location.origin)
+    }
 
     const checked = this.columnCheckboxTargets.filter((cb) => cb.checked)
     if (checked.length === 0) {
       throw new Error(this.needsColumnMessageValue)
     }
-    checked.forEach((cb) => u.searchParams.append("export_columns[]", cb.value))
-    return u.toString()
+    return appendExportColumnParams(
+      urlString,
+      checked.map((cb) => cb.value),
+      window.location.origin,
+    )
+  }
+
+  mimeFor(exportFmt) {
+    if (exportFmt === "csv") return "text/csv;charset=utf-8"
+    if (exportFmt === "json") return "application/json;charset=utf-8"
+    if (exportFmt === "xml") return "application/xml;charset=utf-8"
+    return "application/octet-stream"
+  }
+
+  offerSave(filename) {
+    this.filename = filename
+    this.readyBlob = new Blob(assembleExportParts(this.exportFmt, this.parts), { type: this.mime })
+    if (this.hasSaveTarget) {
+      this.saveTarget.classList.remove("hidden")
+      this.saveTarget.disabled = false
+    }
+  }
+
+  setBusy(busy) {
+    if (this.hasStartTarget) this.startTarget.disabled = busy
+    if (this.hasCancelTarget) {
+      this.cancelTarget.classList.toggle("hidden", !busy)
+      this.cancelTarget.disabled = !busy
+    }
+  }
+
+  cancel() {
+    if (this.abortController) this.abortController.abort()
   }
 
   async start() {
@@ -63,13 +110,16 @@ export default class extends Controller {
     this.parts = []
     this.readyBlob = null
     this.filename = null
+    this.baseFilename = null
+    this.exportFmt = this.formatValue
+    this.abortController = new AbortController()
+    const { signal } = this.abortController
 
-    if (this.hasStartTarget) this.startTarget.disabled = true
+    this.setBusy(true)
     if (this.hasSaveTarget) {
       this.saveTarget.disabled = true
       this.saveTarget.classList.add("hidden")
     }
-
     if (this.hasProgressWrapTarget) this.progressWrapTarget.classList.remove("hidden")
     if (this.hasBarTarget) {
       this.barTarget.value = 0
@@ -79,100 +129,90 @@ export default class extends Controller {
     if (this.hasFractionTarget) this.fractionTarget.textContent = ""
 
     try {
-      const batchBaseUrl = this.urlWithExportColumns(this.batchBaseUrlValue)
-
-      let meta = this.hasMetaValue ? this.metaValue : null
-      if (!meta || typeof meta.total_batches !== "number") {
-        const metaUrl = this.urlWithExportColumns(this.metaUrlValue)
-        const metaRes = await fetch(metaUrl, {
-          credentials: "same-origin",
-          headers: { Accept: "application/json", "X-Requested-With": "XMLHttpRequest" },
-        })
-        if (!metaRes.ok) throw new Error(`${metaRes.status} ${metaRes.statusText}`)
-        meta = await metaRes.json()
-      }
-
-      this.filename = meta.filename
-      const total = meta.total_batches
-      const exportFmt = meta.export_format || this.formatValue
+      await this.runExport(signal)
+    } catch (err) {
+      this.keepPartialFile(err)
+    } finally {
+      this.setBusy(false)
+      this.abortController = null
+    }
+  }
 
-      if (total === 0) {
-        if (this.hasStatusTarget) this.statusTarget.textContent = this.emptyMessageValue
-        if (this.hasBarTarget) this.barTarget.removeAttribute("value")
-        if (this.hasStartTarget) this.startTarget.disabled = false
-        return
-      }
+  async runExport(signal) {
+    const batchBaseUrl = this.urlWithExportColumns(this.batchBaseUrlValue)
+    const metaUrl = this.urlWithExportColumns(this.metaUrlValue)
+    const metaRes = await fetch(metaUrl, {
+      credentials: "same-origin",
+      signal,
+      headers: { Accept: "application/json", "X-Requested-With": "XMLHttpRequest" },
+    })
+    if (!metaRes.ok) throw new Error(this.failedBatchTemplateValue.replace("%{page}", "1").replace("%{message}", "metadata"))
+    const meta = await metaRes.json()
+
+    this.exportFmt = meta.export_format || this.formatValue
+    this.mime = this.mimeFor(this.exportFmt)
+    const total = meta.total_batches
+    this.baseFilename = meta.filename
+
+    if (meta.over_max) throw new Error(this.overMaxMessageValue)
+
+    if (total === 0) {
+      if (this.hasStatusTarget) this.statusTarget.textContent = this.emptyMessageValue
+      if (this.hasBarTarget) this.barTarget.removeAttribute("value")
+      return
+    }
 
-      if (exportFmt === "csv") {
-        this.mime = "text/csv;charset=utf-8"
-      } else if (exportFmt === "json") {
-        this.mime = "application/json;charset=utf-8"
-      } else if (exportFmt === "xml") {
-        this.mime = "application/xml;charset=utf-8"
+    let cursor = null
+    let snapshotToken = null
+    let page = 0
+    while (true) {
+      page += 1
+      if (this.hasStatusTarget) this.statusTarget.textContent = this.loadingLabel(page, total)
+      if (this.hasFractionTarget) this.fractionTarget.textContent = `${page} / ${total}`
+      if (this.hasBarTarget) {
+        this.barTarget.value = Math.min(99, Math.round((100 * page) / Math.max(total, page)))
       }
 
-      const collectedJsonRows = []
-      const xmlFragments = []
-
-      for (let page = 1; page <= total; page += 1) {
-        if (this.hasStatusTarget) this.statusTarget.textContent = this.loadingLabel(page, total)
-        if (this.hasFractionTarget) this.fractionTarget.textContent = `${page} / ${total}`
-
-        const batchUrl = new URL(batchBaseUrl, window.location.origin)
-        batchUrl.searchParams.set("batch_page", String(page))
-
-        const batchRes = await fetch(batchUrl.toString(), {
-          credentials: "same-origin",
-          headers: { Accept: "*/*", "X-Requested-With": "XMLHttpRequest" },
-        })
-        if (!batchRes.ok) {
-          const msg = this.failedBatchTemplateValue
+      const batchUrl = new URL(batchBaseUrl, window.location.origin)
+      batchUrl.searchParams.delete("export_cursor")
+      batchUrl.searchParams.delete("export_snapshot")
+      if (cursor) batchUrl.searchParams.set("export_cursor", cursor)
+      if (snapshotToken) batchUrl.searchParams.set("export_snapshot", snapshotToken)
+
+      const batchRes = await fetch(batchUrl.toString(), {
+        credentials: "same-origin",
+        signal,
+        headers: { Accept: "*/*", "X-Requested-With": "XMLHttpRequest" },
+      })
+      if (!batchRes.ok) {
+        throw new Error(
+          this.failedBatchTemplateValue
             .replace("%{page}", String(page))
-            .replace("%{message}", `${batchRes.status} ${batchRes.statusText}`)
-          throw new Error(msg)
-        }
-        const text = await batchRes.text()
-
-        if (exportFmt === "csv") {
-          this.parts.push(text)
-        } else if (exportFmt === "json") {
-          let chunk
-          try {
-            chunk = JSON.parse(text)
-          } catch (parseErr) {
-            throw new Error(
-              this.failedBatchTemplateValue
-                .replace("%{page}", String(page))
-                .replace("%{message}", parseErr.message || "invalid JSON"),
-            )
-          }
-          collectedJsonRows.push(...chunk)
-        } else if (exportFmt === "xml") {
-          xmlFragments.push(text.trim())
-        }
-
-        if (this.hasBarTarget) this.barTarget.value = Math.round((100 * page) / total)
+            .replace("%{message}", "stopped"),
+        )
       }
+      const text = await batchRes.text()
+      this.parts.push(this.exportFmt === "xml" ? text.trim() : text)
 
-      if (exportFmt === "json") {
-        this.parts = [JSON.stringify(collectedJsonRows)]
-      } else if (exportFmt === "xml") {
-        this.parts = [
-          `<?xml version="1.0" encoding="UTF-8"?>\n<export>\n${xmlFragments.join("\n")}\n</export>\n`,
-        ]
-      }
+      const nextSnapshot = batchRes.headers.get("X-Batched-Export-Snapshot")
+      if (nextSnapshot) snapshotToken = nextSnapshot
+      cursor = batchRes.headers.get("X-Batched-Export-Next")
+      if (!cursor) break
+    }
 
-      this.readyBlob = new Blob(this.parts, { type: this.mime })
-      if (this.hasSaveTarget) {
-        this.saveTarget.classList.remove("hidden")
-        this.saveTarget.disabled = false
-      }
-      if (this.hasStatusTarget) this.statusTarget.textContent = this.readyMessageValue
-    } catch (err) {
-      this.showError(err.message || String(err))
-    } finally {
-      if (this.hasStartTarget) this.startTarget.disabled = false
+    if (this.hasBarTarget) this.barTarget.value = 100
+    this.offerSave(this.baseFilename)
+    if (this.hasStatusTarget) this.statusTarget.textContent = this.readyMessageValue
+  }
+
+  keepPartialFile(err) {
+    const cancelled = err && err.name === "AbortError"
+    if (this.parts.length > 0) {
+      const base = this.baseFilename || "export"
+      this.offerSave(`incomplete-${base}`)
+      if (this.hasStatusTarget) this.statusTarget.textContent = this.incompleteMessageValue
     }
+    this.showError(cancelled ? this.cancelledMessageValue : (err.message || String(err)))
   }
 
   save() {

data/app/assets/javascripts/activeadmin_batched_export/chunk_assembly.mjs

@@ -0,0 +1,30 @@
+export function concatJsonArrayChunks(texts) {
+  const inners = []
+  for (const text of texts) {
+    const trimmed = text.trim()
+    if (trimmed === "" || trimmed === "[]") continue
+    inners.push(trimmed.slice(1, -1))
+  }
+  if (inners.length === 0) return "[]"
+  return `[${inners.join(",")}]`
+}
+
+export function wrapXmlChunks(parts) {
+  return `<?xml version="1.0" encoding="UTF-8"?>\n<export>\n${parts.join("\n")}\n</export>\n`
+}
+
+export function assembleExportParts(exportFmt, parts) {
+  if (exportFmt === "json") return [concatJsonArrayChunks(parts)]
+  if (exportFmt === "xml") return [wrapXmlChunks(parts)]
+  return parts
+}
+
+export function appendExportColumnParams(urlString, columnValues, origin) {
+  const parsed = new URL(urlString, origin)
+  parsed.searchParams.delete("export_columns[]")
+  parsed.searchParams.delete("export_columns")
+  for (const value of columnValues) {
+    parsed.searchParams.append("export_columns[]", value)
+  }
+  return parsed.toString()
+}

data/app/views/active_admin/batched_export/_actions.html.erb

@@ -7,6 +7,15 @@
   >
     <%= t("active_admin.batched_export_page.start") %>
   </button>
+  <button
+    type="button"
+    class="<%= styles[:cancel_button] %> hidden"
+    data-<%= stimulus_controller %>-target="cancel"
+    data-action="<%= stimulus_controller %>#cancel"
+    disabled
+  >
+    <%= t("active_admin.batched_export_page.cancel") %>
+  </button>
   <button
     type="button"
     class="<%= styles[:secondary_button] %>"
@@ -19,7 +28,9 @@
   <%= link_to t("active_admin.batched_export_page.back_to_list"),
         url_for(
           action: :index,
-          params: request.query_parameters.except(:format, :commit, :page, :batch_page, :export_meta)
+          params: request.query_parameters.except(
+            :format, :commit, :page, :batch_page, :export_meta, :export_cursor, :export_snapshot
+          )
         ),
         class: styles[:back_link] %>
 </div>

data/app/views/active_admin/batched_export/_progress.html.erb

@@ -11,4 +11,4 @@
   ></progress>
 </div>
 
-<p class="<%= styles[:error] %>" data-<%= stimulus_controller %>-target="error"></p>
+<p class="<%= styles[:error] %>" role="alert" data-<%= stimulus_controller %>-target="error"></p>

data/app/views/active_admin/batched_export/workspace.html.erb

@@ -13,6 +13,9 @@
   data-<%= stimulus_controller %>-failed-batch-template-value="<%= t("active_admin.batched_export_page.failed_batch") %>"
   data-<%= stimulus_controller %>-ready-message-value="<%= t("active_admin.batched_export_page.ready") %>"
   data-<%= stimulus_controller %>-needs-column-message-value="<%= t("active_admin.batched_export_page.select_at_least_one_column") %>"
+  data-<%= stimulus_controller %>-cancelled-message-value="<%= t("active_admin.batched_export_page.cancelled") %>"
+  data-<%= stimulus_controller %>-incomplete-message-value="<%= t("active_admin.batched_export_page.incomplete") %>"
+  data-<%= stimulus_controller %>-over-max-message-value="<%= t("active_admin.batched_export_page.over_max_rows") %>"
 >
   <%= render "active_admin/batched_export/summary", styles: styles %>
   <%= render "active_admin/batched_export/columns", styles: styles, stimulus_controller: stimulus_controller %>

data/app/views/active_admin/shared/_download_format_links.html.erb

@@ -1,4 +1,4 @@
-<% export_base_params = request.query_parameters.except(:format, :commit, :page, :batch_page, :export_meta) %>
+<% export_base_params = request.query_parameters.except(:format, :commit, :page, :batch_page, :export_meta, :export_cursor, :export_snapshot) %>
 <div>
   <span><%= I18n.t("active_admin.download") %></span>
   <% if active_admin_config.batched_export_enabled? %>

data/config/locales/activeadmin_batched_export.en.yml

@@ -5,15 +5,19 @@ en:
       back_to_list: Back to list
       batch_size_label: Rows per batch
       batches_label: Download batches
-      blurb: The export loads in separate batches (one HTTP request per batch). Rows are not shown on this page. When loading finishes, use Save file to store the export on your computer.
+      blurb: The export loads in separate batches. Rows are not shown on this page. When loading finishes, use Save file to store the export on your computer.
+      cancel: Cancel
+      cancelled: Loading stopped. You can save what loaded.
       columns_hint: All columns are included by default. Uncheck any you do not need.
       columns_title: Columns to include
       empty: There is nothing to export for this filter.
       failed_batch: "Could not load batch %{page}: %{message}"
+      incomplete: Incomplete. You can still save what loaded.
+      over_max_rows: This export is larger than the allowed maximum.
       filename_label: File name
       format_label: File format
       large_export_warning: This export is large. The browser keeps the full file in memory before you save it. Narrow filters or export fewer columns if the tab becomes slow.
-      loading_batch: Loading batch %{current} of %{total}
+      loading_batch: Loading batch %{current} of about %{total}
       preparing: Preparing…
       ready: Ready. You can save the file now.
       resource_label: Resource
@@ -26,4 +30,5 @@ en:
       start: Load export
       summary_title: What you are exporting
       total_rows_label: Rows to export
+      unavailable_session: This export session is no longer available.
       unknown_macro: Unknown export macro

data/lib/activeadmin/batched_export/chunk_renderer.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+require "builder"
+require "csv"
+require "activeadmin/batched_export/row_sanitizer"
+
+module ActiveAdmin
+  module BatchedExport
+    module ChunkRenderer
+      def batched_export_batch_body(export_format, records, first_page:)
+        case export_format
+        when :csv then batched_csv_chunk(records, first_page: first_page)
+        when :json then batched_json_chunk(records)
+        when :xml then batched_xml_chunk(records)
+        else ""
+        end
+      end
+
+      def batched_csv_chunk(records, first_page:)
+        builder = export_builder
+        options = builder.options.dup
+        csv_options = options.except(:encoding_options, :humanize_name, :byte_order_mark)
+        columns = export_columns_for(builder)
+        buffer = csv_opening(builder, columns, options, csv_options, first_page)
+        each_export_row(records) do |resource|
+          row = decorated_export_row(builder, columns, options, resource)
+          buffer << CSV.generate_line(row, **csv_options)
+        end
+        buffer
+      end
+
+      def batched_json_chunk(records)
+        builder = export_builder
+        options = builder.options
+        columns = export_columns_for(builder)
+        names = columns.map(&:name)
+        rows = []
+        each_export_row(records) do |resource|
+          row = decorated_export_row(builder, columns, options, resource)
+          rows << names.zip(row).to_h
+        end
+        rows.to_json
+      end
+
+      def batched_xml_chunk(records)
+        builder = export_builder
+        options = builder.options
+        columns = export_columns_for(builder)
+        xml = Builder::XmlMarkup.new(indent: 0)
+        each_export_row(records) do |resource|
+          append_xml_record(xml, columns, decorated_export_row(builder, columns, options, resource))
+        end
+        xml.target!
+      end
+
+      def sanitize_macro_row(row, columns, resource)
+        RowSanitizer.apply(apply_export_macros(row, columns, resource))
+      end
+
+      def each_export_row(records)
+        records.each { |resource| yield apply_decorator(resource) }
+      end
+
+      def apply_export_macros(row, columns, resource)
+        ExportMacroResolver.apply(
+          row: row,
+          columns: columns,
+          resource: resource,
+          resource_settings: active_admin_config.batched_export_settings,
+          registry: merged_macro_registry
+        )
+      end
+
+      def merged_macro_registry
+        BatchedExport.config.registered_macros.merge(ExportMacroCatalog.global_registry)
+      end
+
+      def export_builder
+        active_admin_config.csv_builder
+      end
+
+      def export_columns_for(builder)
+        batched_export_filter_columns(builder.exec_columns(view_context))
+      end
+
+      def decorated_export_row(builder, columns, options, resource)
+        sanitize_macro_row(builder.build_row(resource, columns, options), columns, resource)
+      end
+
+      def csv_opening(builder, columns, options, csv_options, first_page)
+        buffer = +""
+        mark = options[:byte_order_mark]
+        buffer << mark if first_page && mark
+        return buffer unless first_page && options.fetch(:column_names, true)
+
+        headers = columns.map do |column|
+          ActiveAdmin::Sanitizer.sanitize(builder.send(:encode, column.name, options))
+        end
+        buffer << CSV.generate_line(headers, **csv_options)
+      end
+
+      def append_xml_record(xml, columns, row)
+        xml.batch do
+          xml.record do
+            columns.each_with_index do |column, index|
+              xml.field("name" => column.name) { xml.text!(row[index].to_s) }
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/activeadmin/batched_export/configuration.rb

@@ -6,14 +6,16 @@ require "activeadmin/batched_export/export_macro_catalog"
 module ActiveAdmin
   module BatchedExport
     class Configuration
-      attr_accessor :styles, :batch_size, :max_batch_size, :large_export_row_threshold, :stimulus_controller,
-        :default_enabled, :default_column_selection
+      attr_accessor :styles, :batch_size, :max_batch_size, :large_export_row_threshold, :max_export_rows,
+        :snapshot_ttl, :stimulus_controller, :default_enabled, :default_column_selection
 
       def initialize
         @styles = Styles.new
         @batch_size = 1000
         @max_batch_size = 10_000
         @large_export_row_threshold = 25_000
+        @max_export_rows = nil
+        @snapshot_ttl = 86_400
         @stimulus_controller = "activeadmin-batched-export--batched-export"
         @default_enabled = false
         @default_column_selection = true