patchwork_csv_utils 0.1.22-arm64-darwin → 0.1.24-arm64-darwin

data/Gemfile

@@ -8,7 +8,7 @@ gemspec
 group :development do
   gem 'rake', '~> 13.0'
   gem 'rake-compiler'
-  gem 'rb_sys', '~> 0.9.98'
+  gem 'rb_sys', '~> 0.9.117'
   gem 'rspec', '~> 3.0'
   gem 'rubocop', '~> 1.21'
 end

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    patchwork_csv_utils (0.1.22)
+    patchwork_csv_utils (0.1.24)
 
 GEM
   remote: https://rubygems.org/
@@ -19,7 +19,9 @@ GEM
     rake (13.2.1)
     rake-compiler (1.2.7)
       rake
-    rb_sys (0.9.98)
+    rake-compiler-dock (1.9.1)
+    rb_sys (0.9.117)
+      rake-compiler-dock (= 1.9.1)
     regexp_parser (2.9.2)
     rexml (3.3.2)
       strscan
@@ -63,7 +65,7 @@ DEPENDENCIES
   patchwork_csv_utils!
   rake (~> 13.0)
   rake-compiler
-  rb_sys (~> 0.9.98)
+  rb_sys (~> 0.9.117)
   rspec (~> 3.0)
   rubocop (~> 1.21)
 

data/ext/csv_utils/Cargo.toml

@@ -2,6 +2,7 @@
 name = "csv_utils"
 version = "0.1.0"
 edition = "2021"
+rust-version = "1.83"
 authors = ["kingsley.hendrickse <kingsley.hendrickse@patchwork.health>"]
 publish = false
 
@@ -9,7 +10,7 @@ publish = false
 crate-type = ["cdylib"]
 
 [dependencies]
-magnus = { version = "0.7.1" }
-csv = "1.3.0"
-calamine = {  version = "0.25.0", features = ["dates"] }
-chrono = "0.4.38"
+magnus = { version = "0.8" }
+csv = "1.4"
+calamine = { version = "0.31", features = ["dates"] }
+chrono = "0.4"

data/ext/csv_utils/src/lib.rs

@@ -1,15 +1,15 @@
-use magnus::{define_module, function, prelude::*};
 use crate::utils::csv::transform_csv;
 use crate::utils::dedup::dedup;
 use crate::utils::xls::to_csv;
+use magnus::{function, prelude::*, Ruby};
 
 pub mod utils;
 
 #[magnus::init]
-fn init() -> Result<(), magnus::Error> {
-    let module = define_module("CsvUtils")?;
+fn init(ruby: &Ruby) -> Result<(), magnus::Error> {
+    let module = ruby.define_module("CsvUtils")?;
     module.define_singleton_method("dedup", function!(dedup, 4))?;
-    module.define_singleton_method("to_csv", function!(to_csv, 7))?;
-    module.define_singleton_method("transform_csv", function!(transform_csv, 7))?;
+    module.define_singleton_method("to_csv", function!(to_csv, 8))?;
+    module.define_singleton_method("transform_csv", function!(transform_csv, 8))?;
     Ok(())
 }

data/ext/csv_utils/src/utils/csv.rs

@@ -1,90 +1,150 @@
-use chrono::{NaiveDate, NaiveDateTime, NaiveTime, Utc};
+use chrono::Utc;
 use csv::{Reader, StringRecord, Writer};
 use magnus::{Error, RArray, Ruby};
 use std::collections::HashMap;
 use std::fs::File;
 
-use crate::utils::{check_mandatory_headers, create_header_map, headers_as_byte_record, index_of_header_in_mandatory_list, magnus_err, missing_header, missing_value, to_datetime_error, FileExtension};
-
-pub fn transform_csv(ruby: &Ruby,
-                     csv_path: String,
-                     target_path: String,
-                     exclusions: RArray,
-                     mandatory_headers: RArray,
-                     status_exclusions: RArray,
-                     expected_trust_name: String,
-                     is_streamed_file: bool) -> magnus::error::Result<()> {
+use crate::utils::shared::datetime::DateTimeProcessor;
+use crate::utils::shared::filters::RowFilters;
+use crate::utils::shared::types::{HeaderConfig, MandatoryColumn, ProcessingConfig};
+use crate::utils::shared::validation::TrustValidator;
+use crate::utils::{
+    check_mandatory_headers, create_header_map, headers_as_byte_record,
+    index_of_header_in_mandatory_list, magnus_err, missing_header, missing_value,
+    string_to_datetime, FileExtension,
+};
+
+#[allow(clippy::too_many_arguments)]
+pub fn transform_csv(
+    ruby: &Ruby,
+    csv_path: String,
+    target_path: String,
+    exclusions: RArray,
+    mandatory_headers: RArray,
+    status_exclusions: RArray,
+    expected_trust_name: String,
+    is_streamed_file: bool,
+    earliest_start_date: Option<String>,
+) -> magnus::error::Result<()> {
     if !csv_path.has_extension(&["csv"]) {
-        return Err(Error::new(ruby.exception_standard_error(), "csv_path must be a csv file".to_string()));
+        return Err(Error::new(
+            ruby.exception_standard_error(),
+            "csv_path must be a csv file".to_string(),
+        ));
     }
 
-    let exclusions = RArray::to_vec(exclusions)?;
-    let status_exclusions = RArray::to_vec(status_exclusions)?;
-    let mandatory_headers: Vec<String> = RArray::to_vec(mandatory_headers)?;
+    let config = ProcessingConfig::from_ruby(
+        exclusions,
+        mandatory_headers,
+        status_exclusions,
+        expected_trust_name,
+        is_streamed_file,
+        earliest_start_date,
+    )?;
 
     let csv_file = File::open(csv_path).map_err(|e| magnus_err(ruby, e, "csv_path"))?;
     let mut csv: Reader<File> = Reader::from_reader(csv_file);
     let mut wtr = Writer::from_path(target_path).map_err(|e| magnus_err(ruby, e, "target_path"))?;
-    let headers = csv.headers().map_err(|e| magnus_err(ruby, e, "csv_path headers"))?.clone();
+
+    let headers = csv
+        .headers()
+        .map_err(|e| magnus_err(ruby, e, "csv_path headers"))?
+        .clone();
     let headers_list: Vec<String> = headers.iter().map(|h| h.to_string()).collect();
 
     if let Some(value) =
-        check_mandatory_headers(ruby, &headers_list, &mandatory_headers, "csv") { return value; }
+        check_mandatory_headers(ruby, &headers_list, &config.mandatory_headers, "csv")
+    {
+        return value;
+    }
 
-    let header_map: HashMap<String, usize> = create_header_map(&mandatory_headers);
-    let inverse_header_map: HashMap<usize, String> = mandatory_headers.iter().enumerate().map(|(i, h)| (i, h.to_string())).collect();
+    let header_map = create_header_map(&config.mandatory_headers);
+    let header_config = HeaderConfig::from_header_map(&header_map, ruby)?;
+    let inverse_header_map: HashMap<usize, String> = config
+        .mandatory_headers
+        .iter()
+        .enumerate()
+        .map(|(i, h)| (i, h.to_string()))
+        .collect();
 
-    let csv_headers = headers_as_byte_record(mandatory_headers.clone());
-    wtr.write_byte_record(&csv_headers).map_err(|e| magnus_err(ruby, e, "write_byte_record"))?;
+    wtr.write_byte_record(&headers_as_byte_record(config.mandatory_headers.clone()))
+        .map_err(|e| magnus_err(ruby, e, "write_byte_record"))?;
 
-    let request_id = header_map.get("Request Id").ok_or(missing_header(ruby, "Request Id"))?;
-    let date = header_map.get("Date").ok_or(missing_header(ruby, "Date"))?;
-    let start = header_map.get("Start").ok_or(missing_header(ruby, "Start"))?;
-    let end = header_map.get("End").ok_or(missing_header(ruby, "End"))?;
-    let actual_start = header_map.get("Actual Start").ok_or(missing_header(ruby, "Actual Start"))?;
-    let actual_end = header_map.get("Actual End").ok_or(missing_header(ruby, "Actual End"))?;
-    let status = header_map.get("Status");
-    let trust_name = header_map.get("Trust").ok_or(missing_header(ruby, "Trust"))?;
+    let filters = RowFilters::new(
+        config.exclusions,
+        config.status_exclusions,
+        config.earliest_start_date,
+    );
+    let trust_validator = TrustValidator::new(config.expected_trust_name, config.is_streamed_file);
 
-    let mandatory_records = get_mandatory_records(&ruby, &mut csv, &headers_list, &mandatory_headers)?;
+    let mandatory_records =
+        get_mandatory_records(ruby, &mut csv, &headers_list, &config.mandatory_headers)?;
 
     for (ri, record) in mandatory_records.iter().enumerate() {
-
-        if skip_excluded_rows(request_id, &status, &record, &exclusions) { continue; }
-        if skip_excluded_status_rows(&status, &record, &status_exclusions) { continue; }
-        if has_empty_row_skip(&record) { continue; }
-        if has_empty_first_col_skip_row(&record) { continue; }
+        if filters.should_skip(
+            record,
+            header_config.request_id,
+            header_config.status,
+            header_config.date,
+        ) {
+            continue;
+        }
 
         let mut date_value = Utc::now().naive_utc();
 
-        let mut columns = vec![];
-        for (i, column) in mandatory_headers.iter().enumerate() {
-            let column_index = header_map.get(column).ok_or(missing_header(ruby, column))?;
-            let column_value = record.get(*column_index).ok_or(missing_value(ruby, column))?;
-            let column_value = column_value.trim_end();
-
-            if !is_streamed_file {
-                validate_trust_name(ruby, &expected_trust_name, trust_name, i, &column_value.to_string())?;
-            }
-
-            if i == *date {
-                let current = string_to_datetime(column_value).ok_or(to_datetime_error(ruby, column_value, ri, "Date"))?;
-                date_value = current;
-                columns.push(current.to_string());
-            } else if i == *start || i == *end || i == *actual_start || i == *actual_end {
-                if column_value.is_empty() {
-                    columns.push(column_value.to_string());
+        let columns: Result<Vec<String>, Error> = config
+            .mandatory_headers
+            .iter()
+            .enumerate()
+            .map(|(i, column)| {
+                let column_index = *header_map
+                    .get(column)
+                    .ok_or_else(|| missing_header(ruby, column))?;
+                let column_value = record
+                    .get(column_index)
+                    .ok_or_else(|| missing_value(ruby, column))?
+                    .trim_end();
+
+                if i == header_config.trust_name {
+                    trust_validator.validate(ruby, column_value)?;
+                }
+
+                if header_config.is_date_column(i) {
+                    let current = string_to_datetime(column_value).ok_or_else(|| {
+                        Error::new(
+                            ruby.exception_standard_error(),
+                            format!(
+                                "Could not parse datetime '{}', row: {}, col: Date",
+                                column_value, ri
+                            ),
+                        )
+                    })?;
+                    date_value = current;
+                    Ok(current.to_string())
+                } else if header_config.is_time_column(i) {
+                    if column_value.is_empty() {
+                        Ok(column_value.to_string())
+                    } else {
+                        let column_name = inverse_header_map
+                            .get(&i)
+                            .map(|s| s.as_str())
+                            .unwrap_or("Unknown");
+                        DateTimeProcessor::new(date_value).process_time_column(
+                            ruby,
+                            column_value,
+                            ri,
+                            column_name,
+                        )
+                    }
                 } else {
-                    let column_name = get_column_name(&inverse_header_map, &i);
-                    let current = process_datetime(ruby, ri, date_value, column_value, &column_name)?;
-                    columns.push(current);
+                    Ok(column_value.to_string())
                 }
-            } else {
-                columns.push(column_value.to_string());
-            }
-        }
-        let record = columns.into_iter().collect::<StringRecord>();
-        wtr.write_byte_record(record.as_byte_record()).map_err(|e| magnus_err(ruby, e, "write_byte_record"))?;
+            })
+            .collect();
+
+        let record = columns?.into_iter().collect::<StringRecord>();
+        wtr.write_byte_record(record.as_byte_record())
+            .map_err(|e| magnus_err(ruby, e, "write_byte_record"))?;
     }
 
     wtr.flush().map_err(|e| magnus_err(ruby, e, "flush"))?;
@@ -92,110 +152,43 @@ pub fn transform_csv(ruby: &Ruby,
     Ok(())
 }
 
-
-fn validate_trust_name(ruby: &Ruby, expected_trust_name: &String, trust_name: &usize, i: usize, s: &String) -> magnus::error::Result<()> {
-    if i == *trust_name {
-        let s = s.trim();
-        if s != &expected_trust_name.clone() {
-            return Err(Error::new(ruby.exception_standard_error(), format!("Trust actual name: '{}' is not as expected: '{}'", s, expected_trust_name)));
-        }
-    }
-    Ok(())
-}
-
-fn get_mandatory_records(ruby: &Ruby, csv: &mut Reader<File>, csv_header_list: &Vec<String>, mandatory_headers_list: &Vec<String>) -> magnus::error::Result<Vec<StringRecord>> {
-    let inverse_header_map: HashMap<usize, String> = csv_header_list.iter().enumerate().map(|(i, h)| (i, h.to_string())).collect();
-
-    let mut records = vec![];
-    for row in csv.records() {
-        let row = row.map_err(|e| magnus_err(ruby, e, "record error"))?;
-        let mut columns = vec![];
-        for (i, column_value) in row.iter().enumerate() {
-            let column_name = inverse_header_map.get(&i).ok_or(missing_header(ruby, &i.to_string()))?;
-            if mandatory_headers_list.contains(column_name) {
-                let index = index_of_header_in_mandatory_list(mandatory_headers_list.clone(), column_name.to_string()).unwrap();
-                columns.push(CsvMandatoryColumn::new(column_value.to_string(), index));
-            }
-        }
-        columns.sort_by(|a, b| a.index.cmp(&b.index));
-        let columns = columns.iter().map(|c| c.value.to_string()).collect::<StringRecord>();
-        records.push(columns);
-    }
-
-    Ok(records)
-}
-
-fn process_datetime(ruby: &Ruby, ri: usize, date_value: NaiveDateTime, c: &str, column_name: &String) -> magnus::error::Result<String> {
-    let maybe_correct = correct_datetime(c);
-    if let Some(correct) = maybe_correct {
-        return Ok(correct.to_string());
-    }
-
-    let current_time = string_to_time(c).ok_or(to_datetime_error(ruby, c, ri, column_name))?;
-    let datetime = transform_time_to_datetime(date_value, current_time);
-    Ok(datetime.to_string())
-}
-
-fn get_column_name(inverse_header_map: &HashMap<usize, String>, i: &usize) -> String {
-    let unknown = "Unknown".to_string();
-    let column_name = inverse_header_map.get(&i).unwrap_or(&unknown);
-    column_name.to_string()
+fn get_mandatory_records(
+    ruby: &Ruby,
+    csv: &mut Reader<File>,
+    csv_header_list: &[String],
+    mandatory_headers_list: &[String],
+) -> magnus::error::Result<Vec<StringRecord>> {
+    let inverse_header_map: HashMap<usize, String> = csv_header_list
+        .iter()
+        .enumerate()
+        .map(|(i, h)| (i, h.to_string()))
+        .collect();
+
+    csv.records()
+        .map(|row_result| {
+            let row = row_result.map_err(|e| magnus_err(ruby, e, "record error"))?;
+
+            let mut columns: Vec<MandatoryColumn<String>> = row
+                .iter()
+                .enumerate()
+                .filter_map(|(i, column_value)| {
+                    inverse_header_map.get(&i).and_then(|column_name| {
+                        if mandatory_headers_list.contains(column_name) {
+                            index_of_header_in_mandatory_list(
+                                mandatory_headers_list.to_vec(),
+                                column_name.to_string(),
+                            )
+                            .map(|index| MandatoryColumn::new(column_value.to_string(), index))
+                        } else {
+                            None
+                        }
+                    })
+                })
+                .collect();
+
+            columns.sort_by_key(|c| c.index);
+
+            Ok(columns.into_iter().map(|c| c.value).collect())
+        })
+        .collect()
 }
-
-fn skip_excluded_rows(request_id: &usize, status: &Option<&usize>, r: &StringRecord, exclusions: &Vec<String>) -> bool {
-    if let Some(status_index) = status {
-        if let Some(status) = r.get(**status_index) {
-            if status.eq("Recalled") {
-                return false
-            }
-        }
-    }
-
-    let value = r.get(*request_id).unwrap_or_default();
-    exclusions.contains(&value.to_string())
-}
-
-fn skip_excluded_status_rows(status: &Option<&usize>, r: &StringRecord, exclusions: &Vec<String>) -> bool {
-    status
-        .map(|index| exclusions.contains(&r[*index].to_string()))
-        .unwrap_or(false)
-}
-
-fn string_to_datetime(s: &str) -> Option<NaiveDateTime> {
-    let maybe_correct = correct_datetime(s);
-    if maybe_correct.is_some() { return maybe_correct; }
-
-    NaiveDate::parse_from_str(s, "%d-%b-%y").ok().map(|d| d.and_hms_opt(0, 0, 0)).flatten()
-}
-
-fn correct_datetime(s: &str) -> Option<NaiveDateTime> {
-    NaiveDateTime::parse_from_str(s, "%Y-%m-%d %H:%M:%S").ok()
-}
-
-fn string_to_time(s: &str) -> Option<NaiveTime> {
-    NaiveTime::parse_from_str(s, "%H:%M").ok()
-}
-
-fn transform_time_to_datetime(t1: NaiveDateTime, t2: NaiveTime) -> NaiveDateTime {
-    NaiveDateTime::new(t1.date(), t2)
-}
-
-fn has_empty_first_col_skip_row(record: &StringRecord) -> bool {
-    record[0].is_empty()
-}
-
-fn has_empty_row_skip(record: &StringRecord) -> bool {
-    record.iter().all(|r| r.is_empty())
-}
-
-#[derive(Debug, PartialOrd, PartialEq, Eq, Ord)]
-struct CsvMandatoryColumn {
-    value: String,
-    index: usize,
-}
-
-impl CsvMandatoryColumn {
-    fn new(value: String, index: usize) -> Self {
-        CsvMandatoryColumn { value, index }
-    }
-}

data/ext/csv_utils/src/utils/dedup.rs

@@ -3,92 +3,127 @@ use std::fs::File;
 use csv::{Reader, StringRecord, Writer};
 use magnus::{RArray, Ruby};
 
-use crate::utils::{FileExtension, magnus_err, check_mandatory_headers, create_header_map, missing_header, missing_value, headers_as_byte_record};
-
-pub fn dedup(ruby: &Ruby, previous_csv_path: String,
-             new_csv_path: String,
-             target_path: String,
-             mandatory_headers: RArray,
+use crate::utils::shared::filters::FilterableRecord;
+use crate::utils::{
+    check_mandatory_headers, create_header_map, headers_as_byte_record, magnus_err, missing_header,
+    missing_value, FileExtension,
+};
+
+pub fn dedup(
+    ruby: &Ruby,
+    previous_csv_path: String,
+    new_csv_path: String,
+    target_path: String,
+    mandatory_headers: RArray,
 ) -> magnus::error::Result<()> {
-    if let Some(value) =
-        check_file_extension(ruby, &previous_csv_path, "previous_csv_path") { return value; }
-
-    if let Some(value) =
-        check_file_extension(ruby, &new_csv_path, "new_csv_path") { return value; }
+    validate_csv_extension(ruby, &previous_csv_path, "previous_csv_path")?;
+    validate_csv_extension(ruby, &new_csv_path, "new_csv_path")?;
 
-    let csv1 = File::open(previous_csv_path).map_err(|e| magnus_err(ruby, e, "previous_csv_path"))?;
+    let csv1 =
+        File::open(previous_csv_path).map_err(|e| magnus_err(ruby, e, "previous_csv_path"))?;
     let csv2 = File::open(new_csv_path).map_err(|e| magnus_err(ruby, e, "new_csv_path"))?;
 
     let mandatory_headers: Vec<String> = RArray::to_vec(mandatory_headers)?;
 
-    let mut previous_csv: csv::Reader<File> = csv::Reader::from_reader(csv1);
-    let mut new_csv: csv::Reader<File> = csv::Reader::from_reader(csv2);
+    let mut previous_csv: Reader<File> = Reader::from_reader(csv1);
+    let mut new_csv: Reader<File> = Reader::from_reader(csv2);
 
     let mut wtr = Writer::from_path(target_path).map_err(|e| magnus_err(ruby, e, "target_path"))?;
 
-    let previous_headers = previous_csv.headers().map_err(|e| magnus_err(ruby, e, "previous_csv"))?.clone();
-    let previous_headers_list : Vec<String> = previous_headers.iter().map(|h| h.to_string()).collect();
-    let new_headers = new_csv.headers().map_err(|e| magnus_err(ruby, e, "new_csv"))?.clone();
-    let new_headers_list : Vec<String> = new_headers.iter().map(|h| h.to_string()).collect();
-
-
-    if let Some(value) =
-        check_mandatory_headers(ruby, &previous_headers_list, &mandatory_headers, "previous_csv") { return value; }
-
-    if let Some(value) =
-        check_mandatory_headers(ruby, &new_headers_list, &mandatory_headers, "new_csv") { return value; }
-
-    let csv_headers = headers_as_byte_record(mandatory_headers.clone());
-    wtr.write_byte_record(&csv_headers).map_err(|e| magnus_err(ruby, e, "write_byte_record"))?;
-
-    let previous_mandatory_records = get_records(ruby, &mut previous_csv, previous_headers_list, &mandatory_headers)?;
-    let new_mandatory_records = get_records(ruby, &mut new_csv, new_headers_list, &mandatory_headers)?;
+    let previous_headers = previous_csv
+        .headers()
+        .map_err(|e| magnus_err(ruby, e, "previous_csv"))?
+        .clone();
+    let previous_headers_list: Vec<String> =
+        previous_headers.iter().map(|h| h.to_string()).collect();
+    let new_headers = new_csv
+        .headers()
+        .map_err(|e| magnus_err(ruby, e, "new_csv"))?
+        .clone();
+    let new_headers_list: Vec<String> = new_headers.iter().map(|h| h.to_string()).collect();
+
+    if let Some(err) = check_mandatory_headers(
+        ruby,
+        &previous_headers_list,
+        &mandatory_headers,
+        "previous_csv",
+    ) {
+        return err;
+    }
 
-    for new_record in new_mandatory_records {
-        if !previous_mandatory_records.contains(&new_record) {
-            wtr.write_byte_record(new_record.as_byte_record()).map_err(|e| magnus_err(ruby, e, "write_byte_record"))?;
-        }
+    if let Some(err) =
+        check_mandatory_headers(ruby, &new_headers_list, &mandatory_headers, "new_csv")
+    {
+        return err;
     }
 
+    wtr.write_byte_record(&headers_as_byte_record(mandatory_headers.clone()))
+        .map_err(|e| magnus_err(ruby, e, "write_byte_record"))?;
+
+    let previous_records = get_records(
+        ruby,
+        &mut previous_csv,
+        &previous_headers_list,
+        &mandatory_headers,
+    )?;
+    let new_records = get_records(ruby, &mut new_csv, &new_headers_list, &mandatory_headers)?;
+
+    new_records
+        .into_iter()
+        .filter(|record| !previous_records.contains(record))
+        .try_for_each(|record| {
+            wtr.write_byte_record(record.as_byte_record())
+                .map_err(|e| magnus_err(ruby, e, "write_byte_record"))
+        })?;
+
     wtr.flush().map_err(|e| magnus_err(ruby, e, "flush"))?;
 
     Ok(())
 }
 
-fn get_records(ruby: &Ruby, csv: &mut Reader<File>, csv_headers: Vec<String>, headers: &Vec<String>) -> magnus::error::Result<Vec<StringRecord>> {
-    let header_map = create_header_map(&csv_headers);
-    let mut records = vec![];
-    for record in csv.records() {
-        let record = record.map_err(|e| magnus_err(ruby, e, "record error"))?;
-
-        if has_empty_row_skip(&record) { continue; }
-        if has_empty_first_col_skip_row(&record) { continue; }
-
-        let mut columns = vec![];
-        for column in headers.iter() {
-            let column_index = header_map.get(column).ok_or(missing_header(ruby, column))?;
-            let column_value = record.get(*column_index).ok_or(missing_value(ruby, column))?;
-            columns.push(column_value.trim_end());
-        }
-        let columns = columns.into_iter().collect::<StringRecord>();
-        records.push(columns);
-    }
-    Ok(records)
+fn get_records(
+    ruby: &Ruby,
+    csv: &mut Reader<File>,
+    csv_headers: &[String],
+    headers: &[String],
+) -> magnus::error::Result<Vec<StringRecord>> {
+    let header_map = create_header_map(csv_headers);
+
+    csv.records()
+        .filter_map(|record_result| match record_result {
+            Ok(record) if !record.is_empty() && !record.has_empty_first_column() => {
+                Some(Ok(record))
+            }
+            Ok(_) => None,
+            Err(e) => Some(Err(magnus_err(ruby, e, "record error"))),
+        })
+        .map(|record_result| {
+            let record = record_result?;
+
+            let columns: Result<Vec<&str>, magnus::Error> = headers
+                .iter()
+                .map(|column| {
+                    let column_index = *header_map
+                        .get(column)
+                        .ok_or_else(|| missing_header(ruby, column))?;
+                    record
+                        .get(column_index)
+                        .map(|s| s.trim_end())
+                        .ok_or_else(|| missing_value(ruby, column))
+                })
+                .collect();
+
+            Ok(columns?.into_iter().collect::<StringRecord>())
+        })
+        .collect()
 }
 
-fn check_file_extension(ruby: &Ruby, csv_path: &String, message: &str) -> Option<magnus::error::Result<()>> {
+fn validate_csv_extension(ruby: &Ruby, csv_path: &str, message: &str) -> magnus::error::Result<()> {
     if !csv_path.has_extension(&["csv"]) {
-        return Some(Err(magnus::Error::new(ruby.exception_standard_error(), format!("{} must be a csv file", message))));
+        return Err(magnus::Error::new(
+            ruby.exception_standard_error(),
+            format!("{} must be a csv file", message),
+        ));
     }
-    None
-}
-
-fn has_empty_first_col_skip_row(previous_record: &StringRecord) -> bool {
-    previous_record[0].is_empty()
-}
-
-fn has_empty_row_skip(record: &StringRecord) -> bool {
-    record.iter().all(|r| r.is_empty())
+    Ok(())
 }
-
-