mirroring 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 26eca3d7349f81fc083e5a9f764944932777d067defe4712c31f83ef76d1663f
-  data.tar.gz: 32e2e458c549b1068e975c9f54a4a6b6a0c9d345d6cf4a9405d1c260a1bda74e
+  metadata.gz: 551d83b55c483bde0db966432149621b6ef18109fc83728801a20249aee59144
+  data.tar.gz: 65b72f3109b2fd308da7f8d23ad28cae434b4d1da8542647c06a11511baf858a
 SHA512:
-  metadata.gz: c7d9eea05b26411cbfd2804cb22ae439e74f629dc77c3c5a80f114d8c1b28094d18c666a23e6e3123c6c791ff2c78f28a1a54f1665c2794425ccc2399918f08d
-  data.tar.gz: c08a094f66251000cacc007da45cf53f4c79ed8c7edf9632fa3b8e818f49558aabb9912a055025b616775ef9fb5d1efc6eaef397824ecead6838c800ef849a2b
+  metadata.gz: 97e282e4c09569ed750c527c7d7bb4ff977a83cee1d46884b75b2daca51b1df4dcbaf14a5cfb2a46c3e0b37e2aa932bd82d6624d3505547b167242364085d460
+  data.tar.gz: 6a1ddb6696341e928adad4c74f340c4c2f98aa5d1e9bdd0d0c1eb73a1fecea3de149dfd9518712df3eecc4d504b4bbe6bb12e13cbd14541c118adfe26d872e7d

data/CHANGELOG.md

@@ -4,6 +4,15 @@ All notable changes to this project will be documented in this file.
 
 For more information about changelogs, check [Keep a Changelog](http://keepachangelog.com).
 
+## 0.1.2 - 2026-10-25
+
+* [FEATURE] `CsvFile#stage`, which stages a CSV file whose first line names its fields
+
+  `CsvFile.new('aids-2025.csv').stage('aids_2025', connection)` stages the rows as
+  `AccessFile#stage` does a table's, "Report Number" as report_number, ready for `Model.mirror`.
+  `AccessFile` is now a `CsvFile` that reads each table through mdb-export; what it stages is the
+  same.
+
 ## 0.1.1 - 2026-10-23
 
 * [ENHANCEMENT] `AccessFile#stage` reads the table into one string, a megabyte at a time

data/README.md

@@ -8,6 +8,7 @@ source did and counting them.
 - **`Counted`**: a table written a record at a time, such as a document fetched by URL.
 - **`Run`**: one refresh, in one transaction, the only one running, with how many rows it
   inserted, updated and deleted.
+- **`CsvFile`**: stages a CSV file whose first line names its fields.
 - **`AccessFile`**: stages a table of a Microsoft Access file, read with `mdb-export`.
 
 It needs PostgreSQL.
@@ -59,6 +60,15 @@ Run.record 'engines' do
 end
 ```
 
+A CSV file stages the same way, its rows named by the second argument:
+
+```ruby
+Run.record 'incidents' do
+  staged = CsvFile.new('aids-2025.csv').stage('aids_2025', Incident.connection)
+  Incident.mirror staged, source: 'aids-2025'
+end
+```
+
 A table written a record at a time counts each save and delete in the run at hand:
 
 ```ruby

data/app/models/access_file.rb

@@ -1,51 +1,16 @@
-require 'csv'
-
-# A Microsoft Access database, as DRS and the NTSB publish theirs, read with mdb-export (mdbtools).
-class AccessFile
+# A Microsoft Access database, as DRS and the NTSB publish theirs, each table read as CSV with
+# mdb-export (mdbtools): a table's own id is access_id.
+class AccessFile < CsvFile
   # The mdb-export command, writing dates with their century, which it leaves out by default.
   EXPORT = ['mdb-export', '-D', '%Y-%m-%d', '-T', '%Y-%m-%d %H:%M:%S'].freeze
 
-  # @param path [String] .accdb or .mdb file.
-  def initialize(path)
-    @path = path
-  end
-
-  # Copies a table into a temporary one of text columns, dropped when the transaction commits:
-  # "AD Number" is ad_number, and the table's own id is access_id.
-  # @param table [String] table name in the Access file, such as AD_DATA or events.
-  # @param connection [ActiveRecord::ConnectionAdapters::PostgreSQLAdapter] inside a transaction.
-  # @return [String] temporary table.
-  def stage(table, connection)
-    staged = "staged_#{column table}"
-    IO.popen [*EXPORT, @path, table], 'rb' do |io|
-      columns = CSV.parse_line(io.gets.force_encoding(Encoding::UTF_8)).map { column it }
-      connection.execute "DROP TABLE IF EXISTS #{staged}"
-      connection.execute <<~SQL.squish
-        CREATE TEMP TABLE #{staged} (#{columns.map { %("#{it}" text) }.join ', '}) ON COMMIT DROP
-      SQL
-      copy io, staged, connection.raw_connection
-    end
-    staged
-  end
-
 private
 
+  def read(table, &) = IO.popen([*EXPORT, @path, table], 'rb', &)
+
   def column(name)
-    column = name.strip.downcase.gsub(/[^a-z0-9]+/, '_').delete_prefix('_').delete_suffix('_')
+    column = super
     # An Access table's own counter, named apart from the id Rails gives every row.
     column == 'id' ? 'access_id' : column
   end
-
-  def copy(io, staged, raw)
-    # One string for every megabyte: a new one each, and its copy without null bytes, left garbage
-    # no GC collected before a table ended, some 100 MB over the NTSB's tables.
-    chunk = String.new(capacity: 1 << 20)
-    raw.copy_data "COPY #{staged} FROM STDIN WITH (FORMAT csv)" do
-      while io.read(1 << 20, chunk)
-        # Some narratives carry null bytes, which PostgreSQL cannot keep in text.
-        chunk.delete! "\x00"
-        raw.put_copy_data chunk
-      end
-    end
-  end
 end

data/app/models/csv_file.rb

@@ -0,0 +1,47 @@
+require 'csv'
+
+# A CSV file whose first line names its fields, as a source publishes one.
+class CsvFile
+  # @param path [String] .csv file, or for an AccessFile the .accdb or .mdb file.
+  def initialize(path)
+    @path = path
+  end
+
+  # Copies a table into a temporary one of text columns, dropped when the transaction commits:
+  # "Report Number" is report_number.
+  # @param table [String] what is staged: a name for the CSV file's rows, such as aids_2025, or a
+  #   table of an Access file, such as AD_DATA or events.
+  # @param connection [ActiveRecord::ConnectionAdapters::PostgreSQLAdapter] inside a transaction.
+  # @return [String] temporary table.
+  def stage(table, connection)
+    staged = "staged_#{column table}"
+    read table do |io|
+      columns = CSV.parse_line(io.gets.force_encoding(Encoding::UTF_8)).map { column it }
+      connection.execute "DROP TABLE IF EXISTS #{staged}"
+      connection.execute <<~SQL.squish
+        CREATE TEMP TABLE #{staged} (#{columns.map { %("#{it}" text) }.join ', '}) ON COMMIT DROP
+      SQL
+      copy io, staged, connection.raw_connection
+    end
+    staged
+  end
+
+private
+
+  def read(_table, &) = File.open(@path, 'rb', &)
+
+  def column(name) = name.strip.downcase.gsub(/[^a-z0-9]+/, '_').delete_prefix('_').delete_suffix('_')
+
+  def copy(io, staged, raw)
+    # One string for every megabyte: a new one each, and its copy without null bytes, left garbage
+    # no GC collected before a table ended, some 100 MB over the NTSB's tables.
+    chunk = String.new(capacity: 1 << 20)
+    raw.copy_data "COPY #{staged} FROM STDIN WITH (FORMAT csv)" do
+      while io.read(1 << 20, chunk)
+        # Some narratives carry null bytes, which PostgreSQL cannot keep in text.
+        chunk.delete! "\x00"
+        raw.put_copy_data chunk
+      end
+    end
+  end
+end

data/lib/mirroring/engine.rb

@@ -1,6 +1,6 @@
 module Mirroring
-  # Lends the host its models (Run, Mirrored, Counted, AccessFile), `mirroring key:` on every model,
-  # and the migration of runs (`bin/rails mirroring:install:migrations`).
+  # Lends the host its models (Run, Mirrored, Counted, CsvFile, AccessFile), `mirroring key:` on
+  # every model, and the migration of runs (`bin/rails mirroring:install:migrations`).
   class Engine < ::Rails::Engine
     engine_name 'mirroring'
 

data/lib/mirroring/version.rb

@@ -1,4 +1,4 @@
 module Mirroring
   # The released version, as RubyGems lists it.
-  VERSION = '0.1.1'
+  VERSION = '0.1.2'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mirroring
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - claudiob
@@ -81,6 +81,7 @@ files:
 - app/models/concerns/counted.rb
 - app/models/concerns/mirrored.rb
 - app/models/concerns/mirrored/incoming.rb
+- app/models/csv_file.rb
 - app/models/mirroring/current.rb
 - app/models/run.rb
 - db/migrate/20261016100000_create_runs.rb