patchwork_csv_utils 0.1.4-x86_64-linux → 0.1.6-x86_64-linux

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b9d62eb73016a46a3ec9caac2d43fd5db562a0856831c7f0e544e9c570e8d550
-  data.tar.gz: 1262e0f45c2b8cfe896be3d4bd6f1074db61aa2b252049f5090cec5feba05c1a
+  metadata.gz: 69855b96ed7f617719daa482f940ef230dd9368c15708539666d4a204a62674e
+  data.tar.gz: 4dee6d49af40934270beccc5e42ffdc7d90f240389b56b38a7898b91e5180b8f
 SHA512:
-  metadata.gz: 4fca3c59b6ff0b5cfd1dcc2ad0ae6efdf8005716980c0a1d111bff0190a4fae3c9bc6b560086416c794be89e3eab46ae6ffc34bdfc408a806c4ac07fe1660608
-  data.tar.gz: 28454948d621cd2272b9a6790d735ce7e2d95da51202accb76a3d56f1a5b2b6bc9a274bf7473073bbc99fac25a64de818ed0a1b9839b94dc0893241ff0290c24
+  metadata.gz: 38925fa8dc2d19ab949fb3e9c3626d7ad1fbde105b0936884f58ccc8d4b849c323bfa092cc469edf00521f8cebed084a606504c76e1ee2c4b00943f38f97fb91
+  data.tar.gz: c1a1371b4063cc96573f2ba89ea5fba730400d27c86a62a20331bc9ad503501ead41a3ebacf447372e7f301d164f77aea8a22b5b9f70c081b1dcc2d72a4a63cc

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    patchwork_csv_utils (0.1.4)
+    patchwork_csv_utils (0.1.6)
 
 GEM
   remote: https://rubygems.org/

data/ext/csv_utils/src/lib.rs

@@ -8,6 +8,6 @@ pub mod utils;
 fn init() -> Result<(), magnus::Error> {
     let module = define_module("CsvUtils")?;
     module.define_singleton_method("dedup", function!(dedup, 3))?;
-    module.define_singleton_method("to_csv", function!(to_csv, 2))?;
+    module.define_singleton_method("to_csv", function!(to_csv, 3))?;
     Ok(())
 }

data/ext/csv_utils/src/utils/dedup.rs

@@ -33,12 +33,20 @@ pub fn dedup(ruby: &Ruby, previous_csv_path: String, new_csv_path: String, targe
     let mut previous_records = vec![];
     for previous_record in previous_csv.records() {
         let previous_record = previous_record.map_err(|e| magnus_err(ruby, e, "previous_record"))?;
+
+        if has_empty_row_skip(&previous_record) { continue; }
+        if has_empty_first_col_skip_row(&previous_record) { continue; }
+
         let previous_record = previous_record.into_iter().map(|r| r.trim_end()).collect::<StringRecord>();
         previous_records.push(previous_record)
     }
 
     for new_record in new_csv.records() {
         let new_record = new_record.map_err(|e| magnus_err(ruby, e, "new_record"))?;
+
+        if has_empty_row_skip(&new_record) { continue; }
+        if has_empty_first_col_skip_row(&new_record) { continue; }
+
         let new_record = new_record.into_iter().map(|r| r.trim_end()).collect::<StringRecord>();
         if !previous_records.contains(&new_record) {
             wtr.write_byte_record(new_record.as_byte_record()).unwrap();
@@ -50,4 +58,18 @@ pub fn dedup(ruby: &Ruby, previous_csv_path: String, new_csv_path: String, targe
     Ok(())
 }
 
+fn has_empty_first_col_skip_row(previous_record: &StringRecord) -> bool {
+    if previous_record[0].is_empty() {
+        return true;
+    }
+    false
+}
+
+fn has_empty_row_skip(record: &StringRecord) -> bool {
+    if record.iter().all(|r| r.is_empty()) {
+        return true;
+    }
+    false
+}
+
 

data/ext/csv_utils/src/utils/xls.rs

@@ -4,11 +4,17 @@ use std::io::{BufWriter, Write};
 
 use calamine::{Data, open_workbook, Range, Reader, Xls};
 use chrono::{NaiveDateTime, Utc};
-use magnus::Ruby;
+use magnus::{RArray, Ruby};
 
 use crate::utils::{FileExtension, magnus_err};
 
-pub fn to_csv(ruby: &Ruby, xls_path: String, target_path: String) -> magnus::error::Result<()> {
+pub fn to_csv(ruby: &Ruby, xls_path: String, target_path: String, exclusions: RArray) -> magnus::error::Result<()> {
+    let exclusions = RArray::to_vec(exclusions)?;
+
+    println!("xls_path: {:?}", xls_path);
+    println!("target_path: {:?}", target_path);
+    println!("exclusions: {:?}", exclusions);
+
     if !xls_path.has_extension(&["xls"]) {
         return Err(magnus::Error::new(ruby.exception_standard_error(), "xls_path must be an xls file".to_string()));
     }
@@ -23,14 +29,15 @@ pub fn to_csv(ruby: &Ruby, xls_path: String, target_path: String) -> magnus::err
     let csv_out_file = File::create(target_path.clone()).map_err(|e| magnus_err(ruby, e, format!("could not create csv file: {}", target_path).as_str()))?;
     let mut dest = BufWriter::new(csv_out_file);
 
-    write_csv(ruby, &mut dest, &range, header_map)
+    write_csv(ruby, &mut dest, &range, header_map, exclusions)
 }
 
-fn write_csv<W: Write>(ruby: &Ruby, dest: &mut W, range: &Range<Data>, header_map: HashMap<String, usize>) -> magnus::error::Result<()> {
+fn write_csv<W: Write>(ruby: &Ruby, dest: &mut W, range: &Range<Data>, header_map: HashMap<String, usize>, exclusions: Vec<String>) -> magnus::error::Result<()> {
     let n = range.get_size().1 - 1;
     for (ri, r) in range.rows().enumerate() {
         let mut date_value = Utc::now().naive_utc();
 
+        if skip_excluded_rows(&header_map, r, &exclusions) { continue; }
         if skip_empty_rows(r) { continue; }
         if skip_rows_with_no_request_id(&header_map, r) { continue; }
 
@@ -69,6 +76,16 @@ fn write_csv<W: Write>(ruby: &Ruby, dest: &mut W, range: &Range<Data>, header_ma
     Ok(())
 }
 
+fn skip_excluded_rows(header_map: &HashMap<String, usize>, r: &[Data], exclusions: &Vec<String>) -> bool {
+    if let Some(request_id) = header_map.get("Request Id") {
+        let value = r[*request_id].to_string();
+        if exclusions.contains(&value) {
+            return true;
+        }
+    }
+    false
+}
+
 fn skip_empty_rows(r: &[Data]) -> bool {
     if r.iter().all(|c| c == &Data::Empty) {
         return true;

data/lib/csv_utils/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module CsvUtils
-  VERSION = '0.1.4'
+  VERSION = '0.1.6'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: patchwork_csv_utils
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.1.6
 platform: x86_64-linux
 authors:
 - kingsley.hendrickse
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-08-01 00:00:00.000000000 Z
+date: 2024-08-06 00:00:00.000000000 Z
 dependencies: []
 description: Deduplication of CSV files and XLS to CSV conversion.
 email: