patchwork_csv_utils 0.1.23-x86_64-linux → 0.1.24-x86_64-linux

data/ext/csv_utils/src/utils/xls.rs

@@ -2,270 +2,331 @@ use std::collections::HashMap;
 use std::fs::File;
 use std::io::{BufWriter, Write};
 
-use calamine::{open_workbook_auto, Data, DataType, Range, Reader};
+use calamine::{open_workbook_auto, Data, ExcelDateTime, Range, Reader};
 use chrono::{NaiveDateTime, Timelike, Utc};
 use magnus::{RArray, Ruby};
 
-use crate::utils::{check_mandatory_headers, index_of_header_in_mandatory_list, magnus_err, missing_header, missing_value, string_to_datetime, to_datetime_error, FileExtension};
-
-pub fn to_csv(ruby: &Ruby, xls_path: String,
-              target_path: String,
-              exclusions: RArray,
-              mandatory_headers: RArray,
-              status_exclusions: RArray,
-              expected_trust_name: String,
-              is_streamed_file: bool,
-              earliest_start_date: Option<String>
+use crate::utils::shared::datetime::DateTimeProcessor;
+use crate::utils::shared::filters::RowFilters;
+use crate::utils::shared::types::{HeaderConfig, MandatoryColumn, ProcessingConfig};
+use crate::utils::shared::validation::TrustValidator;
+use crate::utils::{
+    check_mandatory_headers, index_of_header_in_mandatory_list, magnus_err, missing_header,
+    missing_value, FileExtension,
+};
+
+#[allow(clippy::too_many_arguments)]
+pub fn to_csv(
+    ruby: &Ruby,
+    xls_path: String,
+    target_path: String,
+    exclusions: RArray,
+    mandatory_headers: RArray,
+    status_exclusions: RArray,
+    expected_trust_name: String,
+    is_streamed_file: bool,
+    earliest_start_date: Option<String>,
 ) -> magnus::error::Result<()> {
-    if !xls_path.has_extension(&["xls","xlsx"]) {
-        return Err(magnus::Error::new(ruby.exception_standard_error(), "xls_path must be an xls or xlsx file".to_string()));
+    if !xls_path.has_extension(&["xls", "xlsx"]) {
+        return Err(magnus::Error::new(
+            ruby.exception_standard_error(),
+            "xls_path must be an xls or xlsx file".to_string(),
+        ));
     }
 
-    let exclusions = RArray::to_vec(exclusions)?;
-    let mandatory_headers: Vec<String> = RArray::to_vec(mandatory_headers)?;
-    let status_exclusions = RArray::to_vec(status_exclusions)?;
-    
-    let start_date = earliest_start_date
-        .and_then(|date_str| string_to_datetime(&date_str));
-
-    let mut workbook = open_workbook_auto(&xls_path)
-        .map_err(|e| magnus_err(ruby, e, format!("could not open workbook: {}", xls_path).as_str()))?;
-
-    let range = workbook.worksheet_range_at(0)
-        .ok_or(magnus::Error::new(ruby.exception_standard_error(), "no worksheet found in xls".to_string()))
+    let config = ProcessingConfig::from_ruby(
+        exclusions,
+        mandatory_headers,
+        status_exclusions,
+        expected_trust_name,
+        is_streamed_file,
+        earliest_start_date,
+    )?;
+
+    let mut workbook = open_workbook_auto(&xls_path).map_err(|e| {
+        magnus_err(
+            ruby,
+            e,
+            format!("could not open workbook: {}", xls_path).as_str(),
+        )
+    })?;
+
+    let range = workbook
+        .worksheet_range_at(0)
+        .ok_or_else(|| {
+            magnus::Error::new(
+                ruby.exception_standard_error(),
+                "no worksheet found in xls".to_string(),
+            )
+        })
         .and_then(|r| r.map_err(|e| magnus_err(ruby, e, "could not read worksheet range")))?;
 
-    let headers = range.headers().ok_or(magnus::Error::new(ruby.exception_standard_error(), "no headers found in xls".to_string()))?;
+    let headers = range.headers().ok_or_else(|| {
+        magnus::Error::new(
+            ruby.exception_standard_error(),
+            "no headers found in xls".to_string(),
+        )
+    })?;
     let headers_list: Vec<String> = headers.iter().map(|h| h.to_string()).collect();
 
     if let Some(value) =
-        check_mandatory_headers(ruby, &headers_list, &mandatory_headers, "csv") { return value; }
+        check_mandatory_headers(ruby, &headers_list, &config.mandatory_headers, "csv")
+    {
+        return value;
+    }
 
-    let header_map: HashMap<String, usize> = mandatory_headers.iter().enumerate().map(|(i, h)| (h.to_string(), i)).collect();
-    let csv_out_file = File::create(target_path.clone()).map_err(|e| magnus_err(ruby, e, format!("could not create csv file: {}", target_path).as_str()))?;
+    let csv_out_file = File::create(&target_path).map_err(|e| {
+        magnus_err(
+            ruby,
+            e,
+            format!("could not create csv file: {}", target_path).as_str(),
+        )
+    })?;
     let mut dest = BufWriter::new(csv_out_file);
 
-    write_csv(ruby, &mut dest, &range, header_map, exclusions, mandatory_headers, headers_list, status_exclusions, expected_trust_name, is_streamed_file, start_date)
+    write_csv(ruby, &mut dest, &range, config, headers_list)
 }
 
-fn write_csv<W: Write>(ruby: &Ruby, dest: &mut W, range: &Range<Data>,
-                       header_map: HashMap<String, usize>, exclusions: Vec<String>,
-                       mandatory_headers: Vec<String>,
-                       headers_list: Vec<String>,
-                       status_exclusions: Vec<String>,
-                       expected_trust_name: String,
-                       is_streamed_file: bool,
-                       start_date: Option<NaiveDateTime>) -> magnus::error::Result<()> {
-    let n = mandatory_headers.len() - 1;
-    let request_id = header_map.get("Request Id").ok_or(missing_header(ruby, "Request Id"))?;
-    let date = header_map.get("Date").ok_or(missing_header(ruby, "Date"))?;
-    let start = header_map.get("Start").ok_or(missing_header(ruby, "Start"))?;
-    let end = header_map.get("End").ok_or(missing_header(ruby, "End"))?;
-    let actual_start = header_map.get("Actual Start").ok_or(missing_header(ruby, "Actual Start"))?;
-    let actual_end = header_map.get("Actual End").ok_or(missing_header(ruby, "Actual End"))?;
-    let status = header_map.get("Status");
-    let trust_name = header_map.get("Trust").ok_or(missing_header(ruby, "Trust"))?;
-
-    let mandatory_rows = get_mandatory_records(ruby, range, &headers_list, &mandatory_headers)?;
+fn write_csv<W: Write>(
+    ruby: &Ruby,
+    dest: &mut W,
+    range: &Range<Data>,
+    config: ProcessingConfig,
+    headers_list: Vec<String>,
+) -> magnus::error::Result<()> {
+    let n = config.mandatory_headers.len() - 1;
+    let header_map: HashMap<String, usize> = config
+        .mandatory_headers
+        .iter()
+        .enumerate()
+        .map(|(i, h)| (h.to_string(), i))
+        .collect();
+
+    let header_config = HeaderConfig::from_header_map(&header_map, ruby)?;
+    let filters = RowFilters::new(
+        config.exclusions,
+        config.status_exclusions,
+        config.earliest_start_date,
+    );
+    let trust_validator = TrustValidator::new(config.expected_trust_name, config.is_streamed_file);
+
+    let mandatory_rows =
+        get_mandatory_records(ruby, range, &headers_list, &config.mandatory_headers)?;
 
     for (ri, r) in mandatory_rows.into_iter().enumerate() {
-        let mut date_value = Utc::now().naive_utc();
+        if filters.should_skip(
+            &r,
+            header_config.request_id,
+            header_config.status,
+            header_config.date,
+        ) {
+            continue;
+        }
 
-        if skip_rows_before_start_date(&start_date, &r, &date) { continue; }
-        if skip_excluded_rows(&request_id, &status, &r, &exclusions) { continue; }
-        if skip_excluded_status_rows(&status, &r, &status_exclusions) { continue; }
-        if skip_empty_rows(&r) { continue; }
-        if skip_rows_with_no_request_id(&request_id, &r) { continue; }
-        if date_value_is_not_present(&date, &r) {
-            return Err(magnus::Error::new(ruby.exception_standard_error(), format!("Date value is not present in row: {}", ri)));
+        if r.get(header_config.date)
+            .is_none_or(|d| matches!(d, &Data::Empty))
+        {
+            return Err(magnus::Error::new(
+                ruby.exception_standard_error(),
+                format!("Date value is not present in row: {}", ri),
+            ));
         }
 
-        if !is_streamed_file {
-            validate_trust_name(ruby, &expected_trust_name, trust_name, ri, &r)?;
+        if ri > 0 {
+            if let Some(trust_data) = r.get(header_config.trust_name) {
+                trust_validator.validate(ruby, &trust_data.to_string())?;
+            }
         }
 
-        for (i, c) in mandatory_headers.iter().enumerate() {
-            let column_index = header_map.get(c).ok_or(missing_header(ruby, c))?;
-            let c = r.get(*column_index).ok_or(missing_value(ruby, c))?;
+        let mut date_value = Utc::now().naive_utc();
+
+        for (i, c) in config.mandatory_headers.iter().enumerate() {
+            let column_index = *header_map.get(c).ok_or_else(|| missing_header(ruby, c))?;
+            let c = r.get(column_index).ok_or_else(|| missing_value(ruby, c))?;
 
             match *c {
                 Data::Empty => Ok(()),
-                Data::String(ref s) | Data::DurationIso(ref s) => {
-                    handle_commas(dest, s)
-                }
+                Data::String(ref s) | Data::DurationIso(ref s) => handle_commas(dest, s),
                 Data::Float(ref f) => write!(dest, "{}", f),
                 Data::DateTimeIso(ref s) => {
-                    // Normalize the string to ensure manageable precision
-                    let normalized_s = if s.contains('.') {
-                        let parts: Vec<&str> = s.split('.').collect();
-                        format!("{}.{}", parts[0], &parts[1][..std::cmp::min(parts[1].len(), 6)]) // Keep up to 6 fractional seconds
-                    } else {
-                        s.to_string()
-                    };
-
-                    // Attempt to parse the normalized string as a full datetime
-                    let mut current = NaiveDateTime::parse_from_str(&normalized_s, "%Y-%m-%dT%H:%M:%S%.f")
-                        .or_else(|_| {
-                            // If parsing as datetime fails, try parsing as date-only
-                            NaiveDateTime::parse_from_str(&format!("{}T00:00:00", normalized_s), "%Y-%m-%dT%H:%M:%S%.f")
-                        })
-                        .or_else(|_| {
-                            // If parsing as time-only fails, try parsing as time-only
-                            NaiveDateTime::parse_from_str(&format!("1970-01-01T{}", normalized_s), "%Y-%m-%dT%H:%M:%S%.f")
-                        })
-                        .map_err(|_| to_datetime_error(ruby, s, ri, "Date or Time"))?;
-
-                    // Apply the same logic as for Data::DateTime
-                    if i == *date {
-                        date_value = current;
-                    } else if i == *start || i == *end || i == *actual_start || i == *actual_end {
-                        current = transform_time_to_datetime(date_value, current);
-                    }
-
-                    // Round up to the next second if we have any fractional seconds
-                    let adjusted_time = if current.nanosecond() > 0 {
-                        current + chrono::Duration::seconds(1) - chrono::Duration::nanoseconds(current.nanosecond() as i64)
-                    } else {
-                        current
-                    };
-
-                    // Format the output to ensure consistent precision
-                    let formatted_output = adjusted_time.format("%Y-%m-%d %H:%M:%S").to_string();
-                    write!(dest, "{}", formatted_output)
+                    handle_datetime_iso(ruby, dest, s, ri, i, &header_config, &mut date_value)
                 }
                 Data::DateTime(ref d) => {
-                    let mut current = d.as_datetime().ok_or(to_datetime_error(ruby, &d.to_string(), ri, "Date"))?;
-                    if i == *date {
-                        date_value = current;
-                    } else if i == *start || i == *end || i == *actual_start || i == *actual_end {
-                        current = transform_time_to_datetime(date_value, current);
-                    }
-                    write!(dest, "{}", current)
+                    handle_datetime(dest, *d, ri, i, &header_config, &mut date_value)
                 }
                 Data::Int(ref i) => write!(dest, "{}", i),
                 Data::Error(ref e) => write!(dest, "{:?}", e),
                 Data::Bool(ref b) => write!(dest, "{}", b),
-            }.map_err(|e| magnus_err(ruby, e, format!("error writing xls row: {}, column: {}", ri, i).as_str()))?;
-            if i != n {
-                write!(dest, ",").map_err(|e| magnus_err(ruby, e, format!("error writing csv comma for row: {}, column: {}", ri, i).as_str()))?;
             }
-        }
-        write!(dest, "\r\n").map_err(|e| magnus_err(ruby, e, format!("error writing end of line for row: {}", ri).as_str()))?;
-    }
-    Ok(())
-}
+            .map_err(|e| {
+                magnus_err(
+                    ruby,
+                    e,
+                    format!("error writing xls row: {}, column: {}", ri, i).as_str(),
+                )
+            })?;
 
-fn validate_trust_name(ruby: &Ruby, expected_trust_name: &String, trust_name: &usize, ri: usize, r: &Vec<&Data>) -> magnus::error::Result<()> {
-    if ri > 0 {
-        let s = r[*trust_name].to_string();
-        let s = s.trim();
-        if s != expected_trust_name.clone() {
-            return Err(magnus::Error::new(ruby.exception_standard_error(), format!("Trust actual name: '{}' is not as expected: '{}'", s, expected_trust_name)));
+            if i != n {
+                write!(dest, ",").map_err(|e| {
+                    magnus_err(
+                        ruby,
+                        e,
+                        format!("error writing csv comma for row: {}, column: {}", ri, i).as_str(),
+                    )
+                })?;
+            }
         }
+        write!(dest, "\r\n").map_err(|e| {
+            magnus_err(
+                ruby,
+                e,
+                format!("error writing end of line for row: {}", ri).as_str(),
+            )
+        })?;
     }
     Ok(())
 }
 
-fn get_mandatory_records<'a>(ruby: &Ruby, range: &'a Range<Data>, csv_header_list: &Vec<String>, mandatory_headers_list: &Vec<String>) -> magnus::error::Result<Vec<Vec<&'a Data>>> {
-    let inverse_header_map: HashMap<usize, String> = csv_header_list.iter().enumerate().map(|(i, h)| (i, h.to_string())).collect();
-
-    let mut records = vec![];
-    for row in range.rows() {
-        let mut columns = vec![];
-        for (i, column_value) in row.iter().enumerate() {
-            let column_name = inverse_header_map.get(&i).ok_or(missing_header(ruby, &i.to_string()))?;
-            if mandatory_headers_list.contains(column_name) {
-                let index = index_of_header_in_mandatory_list(mandatory_headers_list.clone(), column_name.to_string()).unwrap();
-                columns.push(XlsMandatoryColumn::new(column_value, index));
-            }
-        }
-        columns.sort_by(|a, b| a.index.cmp(&b.index));
-        let columns = columns.iter().map(|c| c.value).collect::<Vec<&Data>>();
-        records.push(columns);
+fn handle_datetime_iso<W: Write>(
+    _ruby: &Ruby,
+    dest: &mut W,
+    s: &str,
+    ri: usize,
+    i: usize,
+    header_config: &HeaderConfig,
+    date_value: &mut NaiveDateTime,
+) -> std::io::Result<()> {
+    let normalized_s = if s.contains('.') {
+        let parts: Vec<&str> = s.split('.').collect();
+        format!(
+            "{}.{}",
+            parts[0],
+            &parts[1][..std::cmp::min(parts[1].len(), 6)]
+        )
+    } else {
+        s.to_string()
+    };
+
+    let mut current = NaiveDateTime::parse_from_str(&normalized_s, "%Y-%m-%dT%H:%M:%S%.f")
+        .or_else(|_| {
+            NaiveDateTime::parse_from_str(
+                &format!("{}T00:00:00", normalized_s),
+                "%Y-%m-%dT%H:%M:%S%.f",
+            )
+        })
+        .or_else(|_| {
+            NaiveDateTime::parse_from_str(
+                &format!("1970-01-01T{}", normalized_s),
+                "%Y-%m-%dT%H:%M:%S%.f",
+            )
+        })
+        .map_err(|_| {
+            std::io::Error::new(
+                std::io::ErrorKind::InvalidData,
+                format!(
+                    "Could not parse datetime '{}', row: {}, col: Date or Time",
+                    s, ri
+                ),
+            )
+        })?;
+
+    if header_config.is_date_column(i) {
+        *date_value = current;
+    } else if header_config.is_time_column(i) {
+        current = DateTimeProcessor::new(*date_value).combine_datetime_parts(current);
     }
 
-    Ok(records)
-}
+    let adjusted_time = if current.nanosecond() > 0 {
+        current + chrono::Duration::seconds(1)
+            - chrono::Duration::nanoseconds(current.nanosecond() as i64)
+    } else {
+        current
+    };
 
-fn date_value_is_not_present(date: &usize, r: &Vec<&Data>) -> bool {
-    r[*date] == &Data::Empty
+    write!(dest, "{}", adjusted_time.format("%Y-%m-%d %H:%M:%S"))
 }
 
-fn skip_excluded_rows(request_id: &usize, status: &Option<&usize>, r: &Vec<&Data>, exclusions: &Vec<String>) -> bool {
-    if let Some(status_index) = status {
-        if let Some(status) = r.get(**status_index) {
-            if let Some(status_str) = status.as_string() {
-                if status_str.eq("Recalled") {
-                    return false
-                }
-            }
-        }
+fn handle_datetime<W: Write>(
+    dest: &mut W,
+    d: ExcelDateTime,
+    _ri: usize,
+    i: usize,
+    header_config: &HeaderConfig,
+    date_value: &mut NaiveDateTime,
+) -> std::io::Result<()> {
+    let mut current = d.as_datetime().ok_or_else(|| {
+        std::io::Error::new(
+            std::io::ErrorKind::InvalidData,
+            format!(
+                "Could not parse datetime '{:?}', row: {}, col: Date",
+                d, _ri
+            ),
+        )
+    })?;
+
+    if header_config.is_date_column(i) {
+        *date_value = current;
+    } else if header_config.is_time_column(i) {
+        current = DateTimeProcessor::new(*date_value).combine_datetime_parts(current);
     }
 
-    let value = r[*request_id].to_string();
-    exclusions.contains(&value.to_string())
-}
-
-fn skip_excluded_status_rows(status: &Option<&usize>, r: &Vec<&Data>, exclusions: &Vec<String>) -> bool {
-    status
-        .map(|index| exclusions.contains(&r[*index].to_string()))
-        .unwrap_or(false)
-}
-
-fn skip_empty_rows(r: &Vec<&Data>) -> bool {
-    r.into_iter().all(|c| c == &&Data::Empty)
+    write!(dest, "{}", current)
 }
 
-fn skip_rows_with_no_request_id(request_id: &usize, r: &Vec<&Data>) -> bool {
-    r[*request_id] == &Data::Empty
-}
-
-fn skip_rows_before_start_date(start_date: &Option<NaiveDateTime>, r: &Vec<&Data>, date_index: &usize) -> bool {
-    if let Some(start_date) = start_date {
-        if let Some(date_data) = r.get(*date_index) {
-            match date_data {
-                Data::DateTime(d) => {
-                    if let Some(date) = d.as_datetime() {
-                        return date <= *start_date;
-                    }
-                }
-                Data::DateTimeIso(s) => {
-                    if let Some(date) = string_to_datetime(s) {
-                        return date <= *start_date;
-                    }
-                }
-                _ => {}
-            }
-        }
-    }
-    false
-}
-
-fn transform_time_to_datetime(t1: NaiveDateTime, t2: NaiveDateTime) -> NaiveDateTime {
-    NaiveDateTime::new(t1.date(), t2.time())
+fn get_mandatory_records<'a>(
+    _ruby: &Ruby,
+    range: &'a Range<Data>,
+    csv_header_list: &[String],
+    mandatory_headers_list: &[String],
+) -> magnus::error::Result<Vec<Vec<&'a Data>>> {
+    let inverse_header_map: HashMap<usize, String> = csv_header_list
+        .iter()
+        .enumerate()
+        .map(|(i, h)| (i, h.to_string()))
+        .collect();
+
+    range
+        .rows()
+        .map(|row| {
+            let mut columns: Vec<MandatoryColumn<&Data>> = row
+                .iter()
+                .enumerate()
+                .filter_map(|(i, column_value)| {
+                    inverse_header_map.get(&i).and_then(|column_name| {
+                        if mandatory_headers_list.contains(column_name) {
+                            index_of_header_in_mandatory_list(
+                                mandatory_headers_list.to_vec(),
+                                column_name.to_string(),
+                            )
+                            .map(|index| MandatoryColumn::new(column_value, index))
+                        } else {
+                            None
+                        }
+                    })
+                })
+                .collect();
+
+            columns.sort_by_key(|c| c.index);
+
+            Ok(columns.into_iter().map(|c| c.value).collect())
+        })
+        .collect()
 }
 
 fn handle_commas<W: Write>(dest: &mut W, s: &str) -> std::io::Result<()> {
-    if s.contains(",") {
-        write!(dest, "{:?}", clean_strings(s).trim_end())
+    let cleaned = s
+        .chars()
+        .map(|c| match c {
+            '\n' => ' ',
+            '\r' | '"' => '\0',
+            _ => c,
+        })
+        .filter(|&c| c != '\0')
+        .collect::<String>();
+
+    if s.contains(',') {
+        write!(dest, "{:?}", cleaned.trim_end())
     } else {
-        write!(dest, "{}", clean_strings(s).trim_end())
-    }
-}
-
-fn clean_strings(s: &str) -> String {
-    s.replace("\n", " ")
-        .replace("\r", "")
-        .replace("\"", "")
-}
-
-struct XlsMandatoryColumn<'a> {
-    value: &'a Data,
-    index: usize,
-}
-
-impl<'a> XlsMandatoryColumn<'a> {
-    fn new(value: &'a Data, index: usize) -> Self {
-        XlsMandatoryColumn { value, index }
+        write!(dest, "{}", cleaned.trim_end())
     }
 }

data/lib/csv_utils/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module CsvUtils
-  VERSION = '0.1.23'
+  VERSION = '0.1.24'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: patchwork_csv_utils
 version: !ruby/object:Gem::Version
-  version: 0.1.23
+  version: 0.1.24
 platform: x86_64-linux
 authors:
 - kingsley.hendrickse
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2025-07-08 00:00:00.000000000 Z
+date: 2025-11-14 00:00:00.000000000 Z
 dependencies: []
 description: Deduplication of CSV files and XLS to CSV conversion.
 email:
@@ -32,6 +32,11 @@ files:
 - ext/csv_utils/src/utils/csv.rs
 - ext/csv_utils/src/utils/dedup.rs
 - ext/csv_utils/src/utils/mod.rs
+- ext/csv_utils/src/utils/shared/datetime.rs
+- ext/csv_utils/src/utils/shared/filters.rs
+- ext/csv_utils/src/utils/shared/mod.rs
+- ext/csv_utils/src/utils/shared/types.rs
+- ext/csv_utils/src/utils/shared/validation.rs
 - ext/csv_utils/src/utils/xls.rs
 - lib/csv_utils.rb
 - lib/csv_utils/2.7/csv_utils.so
@@ -63,7 +68,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.4
+rubygems_version: 3.5.23
 signing_key: 
 specification_version: 4
 summary: Fast CSV utils