cmdtabs 0.1.0 → 0.1.2

data/lib/cmdtabs/cmdtabs_lib.rb

@@ -0,0 +1,351 @@
+#!/usr/bin/env ruby 
+
+
+def load_input_data(input_path, sep="\t", limit=0)
+	if input_path == '-'
+	  input_data = STDIN
+	else
+	  input_data = File.readlines(input_path)
+	end
+	input_data_arr = input_data.map {|line| line = line.chomp.split(sep, limit)}
+	return input_data_arr
+end 
+
+def load_several_files(all_files, sep = "\t", limit=0)
+        loaded_files = {}
+        all_files.each do |file|
+				if FileTest.directory?(file)
+                	STDERR.puts "#{file} is not a valid file"
+                	next
+				end
+                loaded_files[file] = load_input_data(file, sep, limit)
+        end
+        return loaded_files
+end
+
+def index_array(array, col_from=0, col_to=1)
+	indexed_array = {}
+	array.each do |elements|
+		indexed_array[elements[col_from]] = elements[col_to]
+	end
+	return indexed_array
+end
+
+def write_output_data(output_data, output_path=nil, sep="\t")
+	if !output_path.nil?
+		File.open(output_path, 'w') do |out_file|
+			output_data.each do |line|
+				out_file.puts(line.join(sep))
+			end
+		end
+	else
+		output_data.each do |line|
+			STDOUT.puts line.join(sep)		
+		end
+	end
+end
+
+# aggregate_column_data.rb
+def aggregate_column(input_table, col_index, col_agg, sep)
+  aggregated_data = {}
+  aggregated_data_arr = []
+  input_table.each do |fields|
+    key = fields[col_index]
+    val = fields[col_agg]
+    query = aggregated_data[key]
+    if query.nil?
+      aggregated_data[key] = [val]
+    else
+      query << val
+    end
+  end
+  aggregated_data.each do |k, value|
+  	aggregated_data_arr << [k, value.join(sep)]
+  end
+  return aggregated_data_arr
+end
+
+
+# desaggregate_column_data.rb
+def desaggregate_column(input_table, col_index, sep)
+  desaggregated_data = []
+    
+  input_table.each do |fields|
+    aggregated_fields = fields[col_index]
+    aggregated_fields.split(sep).each do |field|
+      record = fields.clone
+      record[col_index] = field
+      desaggregated_data << record
+    end
+  end
+
+  return desaggregated_data
+end
+
+
+# create_metric_table.rb
+def index_metrics(input_data, attributes)
+	n_attrib = attributes.length
+	indexed_metrics = {}
+	metric_names = []
+	input_data.each do |entry|
+		sample_id = entry.shift
+		sample_attributes = entry[0..n_attrib - 1]
+		metric_name, metric = entry[n_attrib..n_attrib + 1]
+		metric_names << metric_name if !metric_names.include?(metric_name)
+		query = indexed_metrics[sample_id]
+		if query.nil?
+			indexed_metrics[sample_id] = {metric_name => metric}
+			attributes.each_with_index do |attrib, i|
+				indexed_metrics[sample_id][attrib] = sample_attributes[i] 
+			end
+		else
+			query[metric_name] = metric
+		end
+	end
+	return metric_names, indexed_metrics
+end
+
+def create_table(indexed_metrics, samples_tag, attributes, metric_names)
+	allTags = attributes + metric_names
+	table_output = []
+	corrupted_records = []
+	indexed_metrics.each do |sample_name, sample_data|
+		record = [sample_name]
+	 	allTags.each do |tag|
+			record << sample_data[tag]
+	 	end
+	 	if record.count(nil) > 0
+	 		warn("Record #{sample_name} is corrupted:\n#{record.inspect}")
+	 		corrupted_records << record
+	 	else
+	 		table_output << record
+	 	end
+	end
+	allTags.unshift(samples_tag)
+	table_output.unshift(allTags) # Add header
+	corrupted_records.unshift(allTags) # Add header
+	return table_output, corrupted_records
+end
+
+
+#intersect_columns.rb
+def load_records(input_file, cols, full)
+	records = {}
+	full_row_of_records = {}
+	input_file.each do |fields|
+		field = cols.map { |c| fields[c]}
+		records[field] = true
+		full_row_of_records[field] = fields if full
+	end
+	return records.keys, full_row_of_records
+end
+
+
+# merge_tabular.rb
+def load_files(files_path)
+	files = {}
+	files_path.each do |file_name|
+		file = []
+		input_table = load_input_data(file_name)
+		input_table.each do |fields|
+			fields.map{|field|
+				if field == ""
+					'-'
+				else
+					field
+				end
+			}
+			next if fields.count('-') == fields.length #skip blank records
+			file << fields
+		end
+		files[file_name] = file
+	end
+	return files
+end
+
+
+def merge_files(files)
+	parent_table = {}
+	table_length = 0
+	files.each do |file_names, file|
+		local_length = 0
+		file.each do |fields|
+			
+			id = fields.shift 
+			local_length = fields.length
+			if !parent_table.has_key?(id)
+				parent_table[id] = Array.new(table_length,'-')
+			elsif parent_table[id].length < table_length
+				parent_table[id].concat(Array.new(table_length-parent_table[id].length,'-'))
+			end
+			parent_table[id].concat(fields)
+			
+		end
+		table_length += local_length			
+	end
+	parent_table_arr = []
+	parent_table.each do |id, fields|
+		diference = table_length - fields.length
+		fields.concat(Array.new(diference,'-')) if diference > 0
+		fields.unshift(id)
+		parent_table_arr << fields
+	end
+	return parent_table_arr
+end
+
+
+# standard_name_replacer.rb
+def name_replaces(tabular_input, sep, cols_to_replace, translation_index, remove_untranslated=false)
+	translated_fields = []
+	untranslated_fields = []
+	tabular_input.each do |fields|
+		cols_to_replace.each do |col|
+			replaced_field = translation_index[fields[col]]
+			if !replaced_field.nil? 
+				fields[col] = replaced_field 
+				translated_fields << fields
+			elsif !remove_untranslated
+				translated_fields << fields
+			else
+				untranslated_fields << fields
+			end
+		end
+	end
+	return translated_fields, untranslated_fields
+end
+
+
+# column_filter
+def merge_and_filter_tables(input_files, options)
+        header = []
+        filtered_table = []
+        options[:cols_to_show] = (0..input_files.first[0].length - 1).to_a if options[:cols_to_show].nil?
+        input_files.each do |filename, file|
+			if options[:header]
+				if header.empty? 
+                    header = file.shift
+				else
+        	        file.shift
+				end 
+			end
+        	filtered_table.concat(filter_columns(file, options)) 
+        end	
+        filtered_table.uniq! if options[:uniq]
+		if !header.empty?
+	        header = shift_by_array_indexes(header, options[:cols_to_show])
+	        filtered_table.unshift(header)
+		end
+        return filtered_table
+end
+
+def parse_column_indices(sep, col_string)
+    cols = col_string.split(sep).map{|col| col.to_i- 1}
+    return cols
+end
+
+def build_pattern(col_filter, keywords)
+    pattern = {}
+    if !col_filter.nil? && !keywords.nil?
+        keys_per_col = keywords.split('%')
+        abort('Number of keywords not equal to number of filtering columns') if keys_per_col.length != col_filter.length
+        col_filter.each_with_index do |col, i|
+            pattern[col] = keys_per_col[i].split('&')
+        end
+    end
+    return pattern
+end
+
+def expanded_match(string, pattern, match_mode)
+    is_match = false
+    is_match = true if string.include?(pattern) && match_mode == 'i'
+    is_match = true if string == pattern && match_mode == 'c'
+    return is_match
+end
+
+def filter(line, all_patterns, search_mode, match_mode, reverse = false)
+    filter = false
+    all_patterns.each do |col, patterns|
+        is_match = false
+        patterns.each do |pattern|
+            is_match = expanded_match(line[col], pattern, match_mode)
+        	break if is_match     
+        end
+        if is_match && search_mode == 's'
+            filter = false
+            break
+        elsif !is_match && search_mode == 'c'
+            filter = true
+            break
+        elsif !is_match
+            filter = true
+        end
+    end
+    if reverse
+        filter = !filter
+    end
+    return filter
+end
+
+def filter_columns(input_table, options)
+	pattern = build_pattern(options[:col_filter], options[:keywords])
+	filtered_table = []
+	input_table.each do |line|
+        if pattern.nil? || !filter(line, pattern, options[:search_mode], options[:match_mode], options[:reverse])
+        	filtered_table << shift_by_array_indexes(line, options[:cols_to_show]) 
+        end
+	end
+	return filtered_table
+end
+
+def shift_by_array_indexes(arr_sub, indexes)
+	subsetted_arr = indexes.map{ |idx| arr_sub[idx]}
+    return subsetted_arr
+end
+
+# table_linker.rb
+def link_table(indexed_linker, tabular_file, drop_line, sep)
+	linked_table = []
+	tabular_file.each do |fields|
+		id = fields.first
+		info_id = indexed_linker[id]
+		if !info_id.nil?
+			linked_table << fields.push(info_id)
+		else
+			linked_table << fields if !drop_line
+		end
+	end
+	return linked_table
+end
+
+# tag_table.rb
+def load_and_parse_tags(tags, sep)
+	parsed_tags = []
+	tags.each do |tag| 
+		if File.exist?(tag)
+			parsed_tags << load_input_data(tag, sep)
+			break
+		else
+			parsed_tags << tag.split(sep)
+		end
+	end
+	return parsed_tags.flatten
+end
+
+def tag_file(input_file, tags, header)
+	taged_file = []
+	input_file.each_with_index do |fields, n_row|
+		next if n_row == 0 && header
+		taged_file << tags.dup.concat(fields)
+	end
+	return taged_file
+end
+
+
+def extract_columns(table, columns2extract)
+	storage = []
+	table.each do |row|
+		storage << shift_by_array_indexes(row, columns2extract)
+	end
+	return storage
+end

data/lib/cmdtabs/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Cmdtabs
-  VERSION = "0.1.0"
+  VERSION = "0.1.2"
 end

data/lib/cmdtabs.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 require_relative "cmdtabs/version"
+require_relative "cmdtabs/cmdtabs_lib"
+
 
 module Cmdtabs
   class Error < StandardError; end

data/test_scripts.sh

@@ -0,0 +1,88 @@
+#!/usr/bin/env bash
+
+source ~soft_bio_267/initializes/init_ruby
+export PATH=`pwd`/bin:$PATH
+test_data=test/data_tests/
+out=output_test_scripts 
+data_to_test=data_test_scripts
+mkdir $out
+
+#aggregate_column_data --------------------------------------------------------------------------
+aggregate_column_data.rb -i $test_data/cluster_genes_dis_desagg -x 1 -s "," -a 2 1> $out/cluster_genes_dis_AGG
+cat $test_data/cluster_genes_dis_desagg | aggregate_column_data.rb -i '-' -x 1 -s "," -a 2 1> $out/cluster_genes_dis_AGG_stdin
+
+#desaggregate_column_data --------------------------------------------------------------------------
+desaggregate_column_data.rb -i $test_data/cluster_genes_dis_agg -x 1 1> $out/cluster_genes_dis_DESAGG
+cat $test_data/cluster_genes_dis_agg| aggregate_column_data.rb -i '-' -x 2 -s "," -a 1 1> $out/cluster_genes_dis_DESAGG_stdin
+
+#create_metric_table --------------------------------------------------------------------------
+create_metric_table.rb $test_data/all_metrics sample $out/metric_table -c TEST_file
+
+#merge_tabular --------------------------------------------------------------------------
+merge_tabular.rb $test_data/disease_gene $test_data/disease_cluster_uniq > $out/merge_disease_cluster_gene
+
+#tag_table --------------------------------------------------------------------------
+tag_table.rb -i $test_data/cluster_stats -t $test_data/tracker 1> $out/tag_table
+tag_table.rb -i $test_data/cluster_stats_header -H -t $test_data/tracker 1> $out/tag_table_header
+
+# intersect_columns.rb --------------------------------------------------------------------------
+##default values
+intersect_columns.rb -a $test_data/disease_cluster -b $test_data/disease_gene -A 1 -B 1 1> $out/intersect_columns_default
+##STDIN a
+cat $test_data/disease_cluster | intersect_columns.rb -a'-' -b $test_data/disease_gene -A 1 -B 1 1> $out/intersect_columns_default_stdin_a
+##STDIN b
+cat $test_data/disease_gene | intersect_columns.rb -a $test_data/disease_cluster -b '-' -A 1 -B 1 1> $out/intersect_columns_default_stdin_b
+##STDIN a y b
+#cat $test_data/disease_cluster, $test_data/disease_gene | intersect_columns.rb $test_data/disease_gene -a'-' -b '-' -A 1 -B 1 > $out/intersect_columns_default_stdin_a_b
+##count = true
+intersect_columns.rb -a $test_data/disease_cluster -b $test_data/disease_gene -A 1 -B 1 -c 1> $out/intersect_columns_count
+##keep = 'ab'
+#intersect_columns.rb -a $test_data/disease_cluster -b $test_data/disease_gene -A 1 -B 1 -k 'ab' > $out/intersect_columns_ab
+##full = true
+intersect_columns.rb -a $test_data/disease_cluster -b $test_data/disease_gene -A 1 -B 1 --full  1> $out/intersect_columns_full
+
+#table_linker --------------------------------------------------------------------------
+#test standard
+table_linker.rb -i $test_data/disease_cluster -l $test_data/disease_gene -o $out/linked_table
+#test reverse 
+table_linker.rb -i $test_data/disease_gene -l $test_data/disease_cluster -o $out/linked_table_2
+#test remove mismatches
+table_linker.rb -i $test_data/disease_cluster -l $test_data/disease_gene -o $out/linked_table_matches --drop
+
+
+#standard_name_replacer --------------------------------------------------------------------------
+standard_name_replacer.rb -i $test_data/disease_cluster -I $test_data/disease_gene -o $out/replaced_name -c 1 -f 1 -t 2 
+#test keep untranslated
+standard_name_replacer.rb -i $test_data/disease_cluster -I $test_data/disease_gene -o $out/replaced_name_untranstaled -c 1 -f 1 -t 2 -u
+
+#excel_to_tabular.rb --------------------------------------------------------------------------
+excel_to_tabular.rb -i $test_data/cluster_genes.xlsx -c 2,3,4 -s 1 -o $out/cluster_genes_from_excel.txt
+
+#column_filter.rb --------------------------------------------------------------------------
+#test column matching hard
+column_filter.rb  -t "$test_data/disease*" -c 1,2 -f 1 -k "MONDO:0008995" -s "c" -m "c" > $out/column_matching_hard
+#test column matching soft
+column_filter.rb  -t "$test_data/disease*" -c 1,2 -f 1 -k "MONDO" -s "c" -m "i" > $out/column_matching_soft
+#test column matching hard various keys 
+column_filter.rb  -t "$test_data/disease*" -c 1,2 -f 1 -k "MONDO:0008995&MONDO:0017999" -s "c" -m "c" > $out/column_matching_hard_various_keys
+#test column matching hard various keys and every columns 
+column_filter.rb  -t "$test_data/disease*" -c 1,2 -f 1,2 -k "MONDO:0008995&MONDO:0017999%53_ref" -s "c" -m "c" > $out/column_matching_hard_various_keys_and_every_columns
+#test column matching hard various keys and every columns to test header 
+column_filter.rb  -t "$test_data/disease*" -c 1,2 -f 1,2 -k "MONDO:0008995&MONDO:0017999%53_ref" -s "c" -m "c" -H > $out/column_matching_hard_various_keys_and_every_columns_header
+#test column matching hard various keys and some columns
+column_filter.rb  -t "$test_data/disease*" -c 1,2 -f 1,2 -k "MONDO:0008995&MONDO:0017999%53_ref" -s "s" -m "c" > $out/column_matching_hard_various_keys_and_some_columns
+#test column matching soft and some columns
+column_filter.rb  -t "$test_data/disease*" -c 1,2 -f 1,2 -k "MONDO%ref" -s "s" -m "i" > $out/column_matching_soft_and_some_columns
+#test column matching soft and every columns
+column_filter.rb  -t "$test_data/disease*" -c 1,2 -f 1,2 -k "MONDO%ref" -s "c" -m "i" > $out/column_matching_soft_and_every_columns
+#test column matching soft 1 column
+column_filter.rb  -t "$test_data/disease*" -c 1 -f 1 -k "MONDO" -s "c" -m "i" > $out/column_matching_soft_1_column
+#test column matching soft 1 column uniq
+column_filter.rb  -t "$test_data/disease*" -c 1 -f 1 -k "MONDO" -s "c" -m "i" -u > $out/column_matching_soft_1_column_uniq 
+#test column matching soft 1 column reverse
+column_filter.rb  -t "$test_data/disease*" -c 1 -f 1 -k "17" -s "c" -m "i" -r > $out/column_matching_soft_1_column_reverse
+
+for file_to_test in `ls $out`; do
+	echo $file_to_test
+	diff $out/$file_to_test $data_to_test/$file_to_test"_to_test"
+done

metadata

@@ -1,27 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: cmdtabs
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.2
 platform: ruby
 authors:
 - seoanezonjic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-03-03 00:00:00.000000000 Z
-dependencies: []
+date: 2025-09-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: xsv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: 'Toolset to merge, colapse tables rename field contents, etc '
 email:
 - seoanezonjic@hotmail.com
 executables:
 - aggregate_column_data.rb
+- column_filter.rb
 - create_metric_table.rb
 - desaggregate_column_data.rb
+- excel_to_tabular.rb
 - intersect_columns.rb
 - merge_tabular.rb
 - standard_name_replacer.rb
-- table_header.rb
 - table_linker.rb
+- tag_table.rb
 extensions: []
 extra_rdoc_files: []
 files:
@@ -34,17 +50,52 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- TEST_file
 - bin/aggregate_column_data.rb
+- bin/column_filter.rb
 - bin/create_metric_table.rb
 - bin/desaggregate_column_data.rb
+- bin/excel_to_tabular.rb
 - bin/intersect_columns.rb
 - bin/merge_tabular.rb
 - bin/standard_name_replacer.rb
-- bin/table_header.rb
 - bin/table_linker.rb
+- bin/tag_table.rb
 - cmdtabs.gemspec
+- data_test_scripts/cluster_genes_dis_AGG_stdin_to_test
+- data_test_scripts/cluster_genes_dis_AGG_to_test
+- data_test_scripts/cluster_genes_dis_DESAGG_stdin_to_test
+- data_test_scripts/cluster_genes_dis_DESAGG_to_test
+- data_test_scripts/cluster_genes_from_excel.txt_to_test
+- data_test_scripts/column_matching_hard_to_test
+- data_test_scripts/column_matching_hard_various_keys_and_every_columns_header_to_test
+- data_test_scripts/column_matching_hard_various_keys_and_every_columns_to_test
+- data_test_scripts/column_matching_hard_various_keys_and_some_columns_to_test
+- data_test_scripts/column_matching_hard_various_keys_to_test
+- data_test_scripts/column_matching_soft_1_column_reverse_to_test
+- data_test_scripts/column_matching_soft_1_column_to_test
+- data_test_scripts/column_matching_soft_1_column_uniq_to_test
+- data_test_scripts/column_matching_soft_and_every_columns_to_test
+- data_test_scripts/column_matching_soft_and_some_columns_to_test
+- data_test_scripts/column_matching_soft_to_test
+- data_test_scripts/intersect_columns_count_to_test
+- data_test_scripts/intersect_columns_default_stdin_a_to_test
+- data_test_scripts/intersect_columns_default_stdin_b_to_test
+- data_test_scripts/intersect_columns_default_to_test
+- data_test_scripts/intersect_columns_full_to_test
+- data_test_scripts/linked_table_2_to_test
+- data_test_scripts/linked_table_matches_to_test
+- data_test_scripts/linked_table_to_test
+- data_test_scripts/merge_disease_cluster_gene_to_test
+- data_test_scripts/metric_table_to_test
+- data_test_scripts/replaced_name_to_test
+- data_test_scripts/replaced_name_untranstaled_to_test
+- data_test_scripts/tag_table_header_to_test
+- data_test_scripts/tag_table_to_test
 - lib/cmdtabs.rb
+- lib/cmdtabs/cmdtabs_lib.rb
 - lib/cmdtabs/version.rb
+- test_scripts.sh
 homepage: https://github.com/seoanezonjic/cmdtabs
 licenses:
 - MIT
@@ -66,8 +117,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.15
+rubygems_version: 3.3.7
 signing_key:
 specification_version: 4
-summary: Gem to manipulate text tables in cmd
+summary: 'DEPRECATED PROJECT. MIGRATED TO PYTHON: https://github.com/seoanezonjic/py_cmdtabs
+  Gem to manipulate text tables in cmd'
 test_files: []