patchwork_csv_utils 0.1.25-aarch64-linux → 0.1.26-aarch64-linux

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 428e2f4abd57105c275a4391288a71fabfc5cae964ba8c7b825027d71164b446
-  data.tar.gz: 7f717a0d5577da392d8f11a71e9117c65762b6da973314443378837c56acd97d
+  metadata.gz: 48b29376c2100ab16451b75344c24acf4e5262436c9063ca37938ddc7c6f82ec
+  data.tar.gz: 1a1a0e368e425b2dbc65fcdfecfa4c871c51914b5c97ee6511f60691f5edfb42
 SHA512:
-  metadata.gz: c9d64bc47d070017c3d5311cb1f0083ca82c6892f6ad644938fa38e3b8bb2e5571952a0c097b0069f15a1e25af6c3bb9e7f08e821763633e382682a741a16fe3
-  data.tar.gz: 0ef2e4c2597e023d8fa85dc6100a1672e54f6250840e668b997c946852cbbb5bec5a7c951cb798f05ba345b7dee97b2d8172681708790a3d1a3215141d414e26
+  metadata.gz: be279537c6c0f636394bb8743ea910865a424a369bcdc868f3df536669b678e0bfed456bd89e0560a60dbb3ca22cc3d23bf5f20b1b0e5f8a2aa58162fa327022
+  data.tar.gz: e66000aa0bb16e83fb2dc9fdc5dd6145865e27a4d5808591b3ce4c107e2ecbf7100478781d335939c7ff4a26933861abc3447002fb0e026dfce8a875d0bfb191

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    patchwork_csv_utils (0.1.25)
+    patchwork_csv_utils (0.1.26)
 
 GEM
   remote: https://rubygems.org/

data/ext/csv_utils/src/utils/csv.rs

@@ -5,7 +5,8 @@ use std::collections::HashMap;
 use std::fs::File;
 
 use crate::utils::shared::datetime::DateTimeProcessor;
-use crate::utils::shared::filters::RowFilters;
+use crate::utils::shared::dropped_rows::dropped_to_ruby;
+use crate::utils::shared::filters::{FilterableRecord, RowFilters, SkipReason};
 use crate::utils::shared::types::{HeaderConfig, MandatoryColumn, ProcessingConfig};
 use crate::utils::shared::validation::TrustValidator;
 use crate::utils::{
@@ -25,7 +26,7 @@ pub fn transform_csv(
     expected_trust_name: String,
     is_streamed_file: bool,
     earliest_start_date: Option<String>,
-) -> magnus::error::Result<()> {
+) -> magnus::error::Result<RArray> {
     if !csv_path.has_extension(&["csv"]) {
         return Err(Error::new(
             ruby.exception_standard_error(),
@@ -55,7 +56,7 @@ pub fn transform_csv(
     if let Some(value) =
         check_mandatory_headers(ruby, &headers_list, &config.mandatory_headers, "csv")
     {
-        return value;
+        return Err(value);
     }
 
     let header_map = create_header_map(&config.mandatory_headers);
@@ -80,13 +81,21 @@ pub fn transform_csv(
     let mandatory_records =
         get_mandatory_records(ruby, &mut csv, &headers_list, &config.mandatory_headers)?;
 
+    let mut dropped: Vec<(String, SkipReason)> = Vec::new();
+
     for (ri, record) in mandatory_records.iter().enumerate() {
-        if filters.should_skip(
+        if let Some(reason) = filters.skip_reason(
             record,
             header_config.request_id,
             header_config.status,
             header_config.date,
         ) {
+            dropped.push((
+                record
+                    .get_request_id(header_config.request_id)
+                    .unwrap_or_default(),
+                reason,
+            ));
             continue;
         }
 
@@ -149,7 +158,7 @@ pub fn transform_csv(
 
     wtr.flush().map_err(|e| magnus_err(ruby, e, "flush"))?;
 
-    Ok(())
+    dropped_to_ruby(ruby, dropped)
 }
 
 fn get_mandatory_records(

data/ext/csv_utils/src/utils/dedup.rs

@@ -48,13 +48,13 @@ pub fn dedup(
         &mandatory_headers,
         "previous_csv",
     ) {
-        return err;
+        return Err(err);
     }
 
     if let Some(err) =
         check_mandatory_headers(ruby, &new_headers_list, &mandatory_headers, "new_csv")
     {
-        return err;
+        return Err(err);
     }
 
     wtr.write_byte_record(&headers_as_byte_record(mandatory_headers.clone()))

data/ext/csv_utils/src/utils/mod.rs

@@ -49,14 +49,14 @@ fn check_mandatory_headers(
     headers: &[String],
     mandatory_headers: &[String],
     message: &str,
-) -> Option<magnus::error::Result<()>> {
+) -> Option<magnus::Error> {
     let csv_mandatory_headers = filter_headers(headers, mandatory_headers);
 
     if csv_mandatory_headers.is_empty() {
-        return Some(Err(magnus::Error::new(
+        return Some(magnus::Error::new(
             ruby.exception_standard_error(),
             format!("{} has no mandatory headers", message),
-        )));
+        ));
     }
 
     let csv_mandatory_headers = csv_mandatory_headers.to_owned().clone();
@@ -71,14 +71,14 @@ fn check_mandatory_headers(
             .iter()
             .map(|h| h.to_string())
             .collect::<Vec<String>>();
-        return Some(Err(magnus::Error::new(
+        return Some(magnus::Error::new(
             ruby.exception_standard_error(),
             format!(
                 "{} is missing mandatory headers: {}",
                 message,
                 missing_headers.join(", ")
             ),
-        )));
+        ));
     }
     None
 }

data/ext/csv_utils/src/utils/shared/dropped_rows.rs

@@ -0,0 +1,25 @@
+use magnus::{RArray, Ruby};
+
+use crate::utils::shared::filters::SkipReason;
+
+/// Converts dropped rows into `[[request_id, reason], ...]` for Ruby.
+///
+/// A flat array of pairs rather than a hash: a Request Id can legitimately appear more
+/// than once in an export, and keying by it would silently collapse those into one.
+///
+/// The id is empty for `empty_row` and `empty_request_id`, because those rows have no
+/// id to report. Consumers should treat those two reasons as counts rather than as
+/// identifiable rows.
+pub fn dropped_to_ruby(
+    ruby: &Ruby,
+    dropped: Vec<(String, SkipReason)>,
+) -> magnus::error::Result<RArray> {
+    let out = ruby.ary_new_capa(dropped.len());
+    for (request_id, reason) in dropped {
+        let pair = ruby.ary_new_capa(2);
+        pair.push(request_id)?;
+        pair.push(reason.as_str())?;
+        out.push(pair)?;
+    }
+    Ok(out)
+}

data/ext/csv_utils/src/utils/shared/filters.rs

@@ -10,6 +10,32 @@ pub trait FilterableRecord {
     fn get_date(&self, index: usize) -> Option<NaiveDateTime>;
 }
 
+/// Why a row was dropped.
+///
+/// The filter used to return a bare bool, so when a shift went missing there was no way
+/// to tell whether Allocate never sent it or Patchwork discarded it, and if discarded,
+/// on which rule. See PLT-2487.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum SkipReason {
+    EmptyRow,
+    EmptyRequestId,
+    PendingTaskExclusion,
+    StatusExclusion,
+    EarliestStartDate,
+}
+
+impl SkipReason {
+    pub fn as_str(self) -> &'static str {
+        match self {
+            SkipReason::EmptyRow => "empty_row",
+            SkipReason::EmptyRequestId => "empty_request_id",
+            SkipReason::PendingTaskExclusion => "pending_task_exclusion",
+            SkipReason::StatusExclusion => "status_exclusion",
+            SkipReason::EarliestStartDate => "earliest_start_date",
+        }
+    }
+}
+
 pub struct RowFilters {
     exclusions: Vec<String>,
     status_exclusions: Vec<String>,
@@ -50,11 +76,36 @@ impl RowFilters {
         status_index: Option<usize>,
         date_index: usize,
     ) -> bool {
-        record.is_empty()
-            || record.has_empty_first_column()
-            || self.should_skip_by_exclusion(record, request_id_index, status_index)
-            || self.should_skip_by_status(record, status_index)
-            || self.should_skip_by_date(record, date_index)
+        self.skip_reason(record, request_id_index, status_index, date_index)
+            .is_some()
+    }
+
+    /// First match wins, preserving the original short-circuit order. A row can satisfy
+    /// several rules at once; reporting the first is enough to explain the drop and
+    /// avoids evaluating predicates whose answer cannot change the outcome.
+    pub fn skip_reason<R: FilterableRecord>(
+        &self,
+        record: &R,
+        request_id_index: usize,
+        status_index: Option<usize>,
+        date_index: usize,
+    ) -> Option<SkipReason> {
+        if record.is_empty() {
+            return Some(SkipReason::EmptyRow);
+        }
+        if record.has_empty_first_column() {
+            return Some(SkipReason::EmptyRequestId);
+        }
+        if self.should_skip_by_exclusion(record, request_id_index, status_index) {
+            return Some(SkipReason::PendingTaskExclusion);
+        }
+        if self.should_skip_by_status(record, status_index) {
+            return Some(SkipReason::StatusExclusion);
+        }
+        if self.should_skip_by_date(record, date_index) {
+            return Some(SkipReason::EarliestStartDate);
+        }
+        None
     }
 
     fn should_skip_by_exclusion<R: FilterableRecord>(
@@ -195,4 +246,120 @@ mod tests {
             2
         ));
     }
+
+    fn reason_for(filters: &RowFilters, cells: &[&str]) -> Option<SkipReason> {
+        filters.skip_reason(&row(cells), 0, Some(1), 2)
+    }
+
+    #[test]
+    fn reports_which_rule_dropped_the_row() {
+        let filters = RowFilters::new(
+            vec!["0726154700".to_string()],
+            vec!["Cancelled".to_string()],
+            Some(
+                NaiveDateTime::parse_from_str("2026-01-01 00:00:00", "%Y-%m-%d %H:%M:%S").unwrap(),
+            ),
+        );
+
+        assert_eq!(
+            reason_for(&filters, &["", "", ""]),
+            Some(SkipReason::EmptyRow)
+        );
+        assert_eq!(
+            reason_for(&filters, &["", "UnFilled Bank", "2026-07-30 00:00:00"]),
+            Some(SkipReason::EmptyRequestId)
+        );
+        assert_eq!(
+            reason_for(
+                &filters,
+                &["0726154700", "UnFilled Bank", "2026-07-30 00:00:00"]
+            ),
+            Some(SkipReason::PendingTaskExclusion)
+        );
+        assert_eq!(
+            reason_for(
+                &filters,
+                &["0726154999", "Cancelled", "2026-07-30 00:00:00"]
+            ),
+            Some(SkipReason::StatusExclusion)
+        );
+        assert_eq!(
+            reason_for(
+                &filters,
+                &["0726154999", "UnFilled Bank", "2025-06-01 00:00:00"]
+            ),
+            Some(SkipReason::EarliestStartDate)
+        );
+    }
+
+    #[test]
+    fn reports_no_reason_for_a_row_that_survives() {
+        let filters = excluding("0726154700");
+
+        assert_eq!(
+            reason_for(
+                &filters,
+                &["0726154999", "UnFilled Bank", "2026-07-30 00:00:00"]
+            ),
+            None
+        );
+    }
+
+    #[test]
+    fn skip_reason_and_should_skip_always_agree() {
+        let filters = RowFilters::new(
+            vec!["0726154700".to_string()],
+            vec!["Cancelled".to_string()],
+            None,
+        );
+
+        for cells in [
+            vec!["", "", ""],
+            vec!["", "UnFilled Bank", "2026-07-30 00:00:00"],
+            vec!["0726154700", "UnFilled Bank", "2026-07-30 00:00:00"],
+            vec!["0726154999", "Cancelled", "2026-07-30 00:00:00"],
+            vec!["0726154999", "UnFilled Bank", "2026-07-30 00:00:00"],
+        ] {
+            let record = row(&cells);
+            assert_eq!(
+                filters.should_skip(&record, 0, Some(1), 2),
+                filters.skip_reason(&record, 0, Some(1), 2).is_some(),
+                "disagreement on {cells:?}"
+            );
+        }
+    }
+
+    #[test]
+    fn reports_the_first_matching_rule_when_several_apply() {
+        // Excluded AND status-excluded. The exclusion check runs first, and reporting one
+        // reason is enough to explain the drop.
+        let filters = RowFilters::new(
+            vec!["0726154700".to_string()],
+            vec!["Cancelled".to_string()],
+            None,
+        );
+
+        assert_eq!(
+            reason_for(
+                &filters,
+                &["0726154700", "Cancelled", "2026-07-30 00:00:00"]
+            ),
+            Some(SkipReason::PendingTaskExclusion)
+        );
+    }
+
+    #[test]
+    fn reason_names_are_stable_for_consumers() {
+        assert_eq!(SkipReason::EmptyRow.as_str(), "empty_row");
+        assert_eq!(SkipReason::EmptyRequestId.as_str(), "empty_request_id");
+        assert_eq!(
+            SkipReason::PendingTaskExclusion.as_str(),
+            "pending_task_exclusion"
+        );
+        assert_eq!(SkipReason::StatusExclusion.as_str(), "status_exclusion");
+        assert_eq!(
+            SkipReason::EarliestStartDate.as_str(),
+            "earliest_start_date"
+        );
+    }
 }

data/ext/csv_utils/src/utils/shared/mod.rs

@@ -1,4 +1,5 @@
 pub mod datetime;
+pub mod dropped_rows;
 pub mod filters;
 pub mod types;
 pub mod validation;

data/ext/csv_utils/src/utils/xls.rs

@@ -7,7 +7,8 @@ use chrono::{NaiveDateTime, Timelike, Utc};
 use magnus::{RArray, Ruby};
 
 use crate::utils::shared::datetime::DateTimeProcessor;
-use crate::utils::shared::filters::RowFilters;
+use crate::utils::shared::dropped_rows::dropped_to_ruby;
+use crate::utils::shared::filters::{FilterableRecord, RowFilters, SkipReason};
 use crate::utils::shared::types::{HeaderConfig, MandatoryColumn, ProcessingConfig};
 use crate::utils::shared::validation::TrustValidator;
 use crate::utils::{
@@ -26,7 +27,7 @@ pub fn to_csv(
     expected_trust_name: String,
     is_streamed_file: bool,
     earliest_start_date: Option<String>,
-) -> magnus::error::Result<()> {
+) -> magnus::error::Result<RArray> {
     if !xls_path.has_extension(&["xls", "xlsx"]) {
         return Err(magnus::Error::new(
             ruby.exception_standard_error(),
@@ -72,7 +73,7 @@ pub fn to_csv(
     if let Some(value) =
         check_mandatory_headers(ruby, &headers_list, &config.mandatory_headers, "csv")
     {
-        return value;
+        return Err(value);
     }
 
     let csv_out_file = File::create(&target_path).map_err(|e| {
@@ -84,7 +85,8 @@ pub fn to_csv(
     })?;
     let mut dest = BufWriter::new(csv_out_file);
 
-    write_csv(ruby, &mut dest, &range, config, headers_list)
+    let dropped = write_csv(ruby, &mut dest, &range, config, headers_list)?;
+    dropped_to_ruby(ruby, dropped)
 }
 
 fn write_csv<W: Write>(
@@ -93,7 +95,7 @@ fn write_csv<W: Write>(
     range: &Range<Data>,
     config: ProcessingConfig,
     headers_list: Vec<String>,
-) -> magnus::error::Result<()> {
+) -> magnus::error::Result<Vec<(String, SkipReason)>> {
     let n = config.mandatory_headers.len() - 1;
     let header_map: HashMap<String, usize> = config
         .mandatory_headers
@@ -113,13 +115,20 @@ fn write_csv<W: Write>(
     let mandatory_rows =
         get_mandatory_records(ruby, range, &headers_list, &config.mandatory_headers)?;
 
+    let mut dropped: Vec<(String, SkipReason)> = Vec::new();
+
     for (ri, r) in mandatory_rows.into_iter().enumerate() {
-        if filters.should_skip(
+        if let Some(reason) = filters.skip_reason(
             &r,
             header_config.request_id,
             header_config.status,
             header_config.date,
         ) {
+            dropped.push((
+                r.get_request_id(header_config.request_id)
+                    .unwrap_or_default(),
+                reason,
+            ));
             continue;
         }
 
@@ -184,7 +193,7 @@ fn write_csv<W: Write>(
             )
         })?;
     }
-    Ok(())
+    Ok(dropped)
 }
 
 fn handle_datetime_iso<W: Write>(

data/lib/csv_utils/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module CsvUtils
-  VERSION = '0.1.25'
+  VERSION = '0.1.26'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: patchwork_csv_utils
 version: !ruby/object:Gem::Version
-  version: 0.1.25
+  version: 0.1.26
 platform: aarch64-linux
 authors:
 - kingsley.hendrickse
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2026-07-24 00:00:00.000000000 Z
+date: 2026-08-13 00:00:00.000000000 Z
 dependencies: []
 description: Deduplication of CSV files and XLS to CSV conversion.
 email:
@@ -33,6 +33,7 @@ files:
 - ext/csv_utils/src/utils/dedup.rs
 - ext/csv_utils/src/utils/mod.rs
 - ext/csv_utils/src/utils/shared/datetime.rs
+- ext/csv_utils/src/utils/shared/dropped_rows.rs
 - ext/csv_utils/src/utils/shared/filters.rs
 - ext/csv_utils/src/utils/shared/mod.rs
 - ext/csv_utils/src/utils/shared/types.rs