immunoscore_results_aggregator 0.0.3

data/lib/data_struct.rb

@@ -0,0 +1,280 @@
+#Copyright (c) 2014, Carlo B. Bifulco. All rights reserved.
+
+require_relative "analyzer"
+require_relative "immunoscore_results_loader"
+
+require_relative "database_connection"
+
+# module Show
+#   def show file_path=self.path
+#     tf=Tempfile.new ["temp",File.extname(file_path)]
+#     tf.write (File.read file_path)
+#     tf.close
+#     puts "#{tf.path}"
+#     fork do
+#       `open #{tf.path}`
+
+#     end
+#     puts Process.waitall
+#     tf.unlink
+#   end
+# end
+
+### Enables all Mongo classes listed below to have instances capable their own CD quicly
+module CdIncluder
+  def get_cd
+    puts self.cd_type
+    if (self.cd_type and self.case_n) != nil 
+      cd= Cd.get(self.cd_type, self.case_n)
+    end
+    puts cd
+    if cd
+      self.cd=cd
+    end
+    self.save
+  end
+end
+
+### enables loading and showing of binary datatsets
+module BinManager
+  def load_bin
+    self.data_load=File.read path
+  end
+  
+  def show graphic_data=self.data_load
+    tf=Tempfile.new ["temp",".jpg"]
+    tf.write graphic_data
+    tf.close
+    puts "#{tf.path}"
+    fork do
+      `open #{tf.path}`
+    end
+    Process.wait
+    tf.unlink
+  end
+end
+
+class ImmunoScoreResults
+  include MongoMapper::Document
+  include DataUtilities
+  safe
+  timestamps!
+  many :cd
+  key :case_n, String
+
+  def self.get case_n
+    self.find_by_case_n(case_n) or self.create(:case_n=>case_n)
+  end
+
+end
+
+
+class Cd
+  include MongoMapper::Document
+  include DataUtilities
+  safe
+  timestamps!
+  belongs_to :immuno_score_results
+  key :cd_type, String
+  key :case_n, String
+  many :ct_tile
+  many :im_tile
+  many :statistic
+  many :classification
+  many :original
+  many :density
+  many :histogram
+
+  #either get a brand new cd with its own Immunoscore Results instance of find a preexisting one
+  def self.get cd_type, case_n
+    self.find_by_cd_type_and_case_n(cd_type,case_n) or self.create(:case_n=>case_n, 
+                                                                      :cd_type=>cd_type,
+                                                                      :immuno_score_results_id=>ImmunoScoreResults.get(case_n)._id)
+  end
+end
+
+
+
+
+  # {:cd=>"CD3",
+  # :path=>
+  #  "/Volumes/LaCie/patient 1-5/patient 1-5/results/RS_SM02-2576-A4_CD3_2013-11-01 16_52_41.image_0000006553/View_Exports
+  # :tile=>"CT2",
+  # :case_n=>"SM02-2576-A4",
+  # :type=>:ct_tile}}
+
+class CtTile
+  include MongoMapper::Document
+  include DataUtilities
+  include CdIncluder
+  include BinManager
+  extend ClassDataUtilities
+  safe
+  timestamps!
+  belongs_to :cd
+  key :tile, String
+  key :path, String
+  key :case_n, String
+  key :cd_type, String
+  key :data_load, Binary
+  before_save :load_bin
+end
+
+
+
+class ImTile
+  include MongoMapper::Document
+  include DataUtilities
+  include CdIncluder
+  include BinManager
+  extend ClassDataUtilities
+  safe
+  timestamps!
+  belongs_to :cd
+  key :tile, String
+  key :path, String
+  key :case_n, String
+  key :cd_type, String
+  key :data_load, Binary
+  before_save :load_bin
+end
+
+# {:case=>"SM02-2456-A6",
+#   :cd=>"CD8",
+#   :path=>
+#    "/Volumes/LaCie/patient 1-5/patient 1-5/results/RS_SM02-2456-A6_CD8_2013-11-01 14_27_27.image_0000006515/image_000000
+#   :type=>:classification},
+class Classification
+  include MongoMapper::Document
+  include DataUtilities
+  include CdIncluder
+  include BinManager
+  extend ClassDataUtilities
+  safe
+  timestamps!
+  belongs_to :cd
+  key :path, String
+  key :case_n, String
+  key :cd_type, String
+  key :data_load, Binary
+  before_save :load_bin
+end
+
+class Original
+  include MongoMapper::Document
+  include DataUtilities
+  include CdIncluder
+  include BinManager
+  extend ClassDataUtilities
+  safe
+  timestamps!
+  belongs_to :cd
+  key :path, String
+  key :case_n, String
+  key :cd_type, String
+  key :data_load, Binary
+  before_save :load_bin
+end
+
+ # {:case=>"RS_SM02-2576-A4_CD3_2013-11-01 16_52_41.image_0000006553",
+ #  :path=>
+ #   "/Volumes/LaCie/patient 1-5/patient 1-5/results/RS_SM02-2576-A4_CD3_2013-11-01 16_52_41.image_0000006553/Statistic_Exports/RS_SM02-2576-A4_CD3_2013-11-01 16_52_41.image_0000006553_Statistics.csv",
+ #  :cd=>"CD3",
+ #  :type=>:statistics}
+
+class Statistic
+  include MongoMapper::Document
+  include DataUtilities
+  include CdIncluder
+  include BinManager
+  extend ClassDataUtilities
+  safe
+  timestamps!
+  belongs_to :cd
+  key :path, String
+  key :case_n, String
+  key :cd_type, String
+  key :data_load, Binary
+  before_save :load_bin
+end
+
+
+ # {:path=>
+ #   "/Volumes/LaCie/patient 1-5/patient 1-5/results/RS_SM02-2576-A4_CD3_2013-11-01 16_52_41.image_0000006553/View_Exports
+ #  :case=>"SM02-2576-A4",
+ #  :new_old=>"old",
+ #  :cd=>"CD3",
+ #  :type=>"density"}
+class Density
+  include MongoMapper::Document
+  include DataUtilities
+  include CdIncluder
+  include BinManager
+  extend ClassDataUtilities
+  safe
+  timestamps!
+  belongs_to :cd
+  key :path, String
+  key :case_n, String
+  key :cd_type, String
+  key :new_old, String
+  key :data_load, Binary
+  before_save :load_bin
+end
+
+
+ # {:path=>
+ #   "/Volumes/LaCie/patient 1-5/patient 1-5/results/RS_SM02-2366-A5_CD8_2013-11-01 16_55_57.image_0000006556/View_Exports
+ #  :case=>"SM02-2366-A5",
+ #  :type=>:histogram,
+ #  :cd=>"CD8"}
+class Histogram
+  include MongoMapper::Document
+  include DataUtilities
+  include CdIncluder
+  include BinManager
+  extend ClassDataUtilities
+  safe
+  timestamps!
+  belongs_to :cd
+  key :path, String
+  key :case_n, String
+  key :cd_type, String
+  key :data_load, Binary
+  before_save :load_bin
+end
+
+
+class Test
+  include MongoMapper::Document
+  include DataUtilities
+  include CdIncluder
+  include BinManager
+  extend ClassDataUtilities
+ 
+  key :path, String
+  
+  key :data_load, Binary
+  before_save :load_bin
+
+
+end
+
+
+
+### mapping from definiens file type to proper database entity
+JSON_CLASS_MAPPER={:ct_tile=>CtTile,
+  :im_tile=>ImTile,
+  :classification=>Classification,
+  :original=>Original,
+  :statistic=>Statistic,
+  :density=>Density,
+  :histogram=>Histogram}
+
+
+### resets all databases
+def mm_clean_all
+  [ImmunoScoreResults, Cd, Histogram, Density, Statistic, Original, Classification, ImTile, CtTile].each do |mm_class|
+    mm_class.delete_all
+  end
+end

data/lib/database_connection.rb

@@ -0,0 +1,4 @@
+require "yaml"
+require "mongo_mapper"
+require_relative "../config.rb"
+MongoMapper.database = DATABASE_NAME

data/lib/exporter.rb

@@ -0,0 +1,76 @@
+#Copyright (c) 2014, Carlo B. Bifulco. All rights reserved.
+
+
+require "csv"
+require_relative "analyzer"
+require_relative "data_struct"
+require_relative "immunoscore_results_loader"
+require_relative "../config.rb"
+
+
+
+def export_clean_all
+  `rm -rf #{EXPORT_DIR}/*`
+end
+
+def cd3? file_path
+  not (file_path !~ /_CD3_/)
+end
+
+def cd8? file_path
+  not (file_path !~ /_CD8_/)
+end
+
+def export_file case_dir,file_path
+  if cd8? file_path
+    Dir.mkdir(case_dir+"/CD8")  unless Dir.exist?(case_dir+"/CD8")
+    fh=File.new(case_dir+"/CD8/"+File.basename(file_path),"w")
+  else
+    Dir.mkdir(case_dir+"/CD3")  unless Dir.exist?(case_dir+"/CD3")
+    fh=File.new(case_dir+"/CD3/"+File.basename(file_path),"w")
+  end
+
+end
+
+# exports novel directory structure with only relevant files
+def export_mongo
+  # defined in immunoscore_results_loader
+  JSON_CLASS_MAPPER.values.each do |mm_class|
+    mm_class.all.each do |mm_object|
+      puts "#{mm_object}: #{mm_object[:case_n]}"
+      next unless mm_object[:case_n] != nil
+      case_dir=(EXPORT_DIR+"/"+ mm_object[:case_n]) 
+      Dir.mkdir case_dir unless Dir.exist?(case_dir)
+      fh=export_file case_dir, mm_object[:path]
+      fh.write(mm_object[:data_load])
+      fh.close
+   
+    end
+  end
+end
+
+
+
+def make_html reporting_order=[
+                :histogram,
+                :original,
+                :density,
+                :ct_tile,
+                :im_tile,
+                :statistic,
+                :classification]
+  all_cases=[]
+  ImmunoScoreResults.all.each do |i|
+    case_summary=[]
+    i.cd.sort.each do |slide_cd|
+      reporting_order.each do |feature|
+        slide_cd.public_send(feature).sort!{|a,b| a.path<=>b.path}.each do |report|
+          #binding.pry
+          case_summary<< report[:path]
+        end
+      end
+    end
+    all_cases << case_summary
+  end
+  all_cases
+end

data/lib/immunoscore_results_aggregator/version.rb

@@ -0,0 +1,3 @@
+module ImmunoscoreTest
+  VERSION = "0.0.1"
+end

data/lib/immunoscore_results_loader.rb

@@ -0,0 +1,218 @@
+#Copyright (c) 2014, Carlo B. Bifulco. All rights reserved.
+
+require "csv"
+require "yaml"
+require "mongo_mapper"
+require_relative "database_connection"
+require_relative "analyzer"
+require_relative "data_struct"
+require_relative "../config.rb"
+
+#puts "WORKING #{`pwd`}; exists #{File.exists?("../config.yaml")}"
+
+
+###Pull info from Definiens Immunoscore Directories
+# 
+# pattern matching based
+
+
+# filename convetions
+# RS_SP02-15779-A8__CD3_2013...
+# no underscores in the file name after the RS prefix
+# they are required before and after case number for pattern matching
+
+#### A few examples of what we are looking for
+#
+#image_0000006499_Classification.jpg
+#image_0000006504_Original.jpg
+#.image_0000006506_densitymap_new.jpg
+#.image_0000006506_histogram.jpg
+#image_0000006506_CT1.jpg. CT2, CT3
+#IM1, IM2, IM3
+
+
+#BASE_DIR="/Volumes/I\$/Christopher\ Paustian/Colon\ Immunoscore"
+
+### regex utilities
+def pull_cd path
+  path.match(/.*(CD[8|3]).*/)[1] if path.match(/.*(CD[8|3]).*/)
+end
+
+def pull_ct path
+  pattern=/.*(CT[1|2|3|4|5|6|7|8])\.jpg/
+  path.match(pattern)[1] if path.match(pattern)
+end
+
+def pull_im path
+  pattern=/.*(IM[1|2|3|4|5|6|7|8])\.jpg/
+  path.match(pattern)[1] if path.match(pattern)
+end
+
+#### Case identification
+# matched file names staring with RS follow by either - or _ followed by SP03
+#followed by either - or _ followed by case number.  Matching of teh block
+# number is not included 
+def get_case_n path
+  match=path.match(/(RS[_-].?.?.?.?.[-_]\d*)/i)
+  match[0] if match
+end
+
+
+### core function
+def find_files file_name_ending="*Classification.jpg", base_dir=BASE_DIR
+  results=[]
+  Dir.glob("#{base_dir}/**/#{file_name_ending}").each do |full_path|
+    puts "am  working on #{full_path}"
+    directory_name=full_path.split("\/")[-4]
+    case_name=get_case_n full_path
+    cd=directory_name.split("_")[2]
+    results << {:case_n=>case_name,
+                :cd_type =>cd,
+                :path=>full_path}
+  end
+  results
+end
+
+
+#### special functions
+
+def find_ct 
+  find_files(file_name_ending="*image*CT*.jpg").map{|x| x[:path]}.map do |path|
+    case_n=get_case_n path
+    cd =pull_cd(path)
+    tile=pull_ct path
+    {:cd_type=>cd, :path=>path, :tile=>tile, :case_n=>case_n, :type=>:ct_tile}
+  end
+end
+
+def find_im
+  find_files(file_name_ending="*image*IM*.jpg").map{|x| x[:path]}.map do |path|
+    case_n=get_case_n path
+    cd =pull_cd(path)
+    tile=pull_im path
+    {:cd_type=>cd, :path=>path, :tile=>tile, :case_n=>case_n,:type=>:im_tile}
+  end
+end
+
+def find_classification
+  find_files(file_name_ending="*Classification.jpg").map do |x|
+    x.merge({:type => :classification})
+  end
+end
+
+def find_original
+  find_files(file_name_ending="*Original.jpg").map do |x|
+     x.merge({:type => :original})
+   end
+end
+
+def find_statistics
+  find_files(file_name_ending="*Statistics.csv").map do |x|
+    puts x
+    {:case_n=> x[:case_n],
+     :path=> x[:path],
+     :cd_type=>pull_cd(x[:path]),
+    :type=> :statistic}
+  end
+end
+
+
+def find_density
+   find_files(file_name_ending="*densitymap*.jpg").map do |x|
+    path=x[:path]
+    case_n=get_case_n path
+    new_old=path.gsub(".jpg","")[-3..-1]
+    {:path=>path, :case_n=>case_n, :new_old=>new_old, 
+      :cd_type=>pull_cd(path),:type=>:density}
+  end
+end
+
+def find_histogram
+  find_files(file_name_ending="*histogram.jpg").map do |x|
+    path=x[:path]
+    case_n=get_case_n path
+    cd=pull_cd(path)
+    {:path=>path, :case_n=>case_n,:type=>:histogram, :cd_type=>cd}
+  end
+end
+
+
+#### merges all the JSON structures coming from the search functions
+# some metaprogramming: calls all find functions listed above
+def search_all
+  r=[]
+  [:find_histogram,:find_density,:find_statistics,:find_original, :find_classification, :find_im,:find_ct].each do |m|
+    r << self.send(m)
+    puts "Done with #{m}"
+  end
+  r.flatten
+end
+
+
+class NilClass
+  def  to_sym
+    false
+  end
+end
+
+
+### query to ensure that entries are not recreated in teh database, but only paths and blobs are updated
+def make_query data_set
+  if data_set.has_key?(:new_old)
+    query={:case_n => data_set[:case_n],:cd_type=>data_set[:cd_type], :new_old=>data_set[:new_old]}
+  elsif data_set.has_key?(:tile)
+    query={:case_n => data_set[:case_n],:cd_type=>data_set[:cd_type], :tile=>data_set[:tile]}
+  else
+    query={:case_n => data_set[:case_n],:cd_type=>data_set[:cd_type]}
+  end
+  query
+end
+
+
+#### load datasets in their mongo classes
+#
+# if entry preexisting updates results
+#
+#
+def mongo_load_all search_results=search_all(), json_class_mapper=JSON_CLASS_MAPPER
+  search_results.each_with_index do |data_set,i|
+    puts "#{i}: #{data_set}"
+    puts data_set["type"]
+    mm_class=( json_class_mapper[data_set["type"]].to_sym or json_class_mapper[data_set[:type]])
+    puts "mm_class=#{mm_class}"
+    #puts "data set type=#{data_set[:type]}"
+    puts "class =#{mm_class}"
+    query=make_query data_set
+    puts "query= #{query}"
+    #query= {:case_n=>"RS-SV-05-16335", :cd_type=>"CD8", :tile=>"CT4"}
+    if mm_class.where(query).all.empty?
+      mm_object=mm_class.create data_set
+      puts "mm created #{mm_object}"
+    else
+      # upserts if entry pre-existing
+      mm_class.set(query, data_set, :upsert => true )
+      puts "uspsert created #{mm_object}"
+      mm_object=mm_class.where(data_set).find_one
+    end
+    #binding.pry
+    mm_object.get_cd
+    mm_object.save
+   
+  end
+  puts "\n\n\n"
+  puts "finished uploading to databes"
+end
+
+def show graphic_data
+  tf=Tempfile.new ["temp",".jpg"]
+  tf.write graphic_data
+  tf.close
+  puts "#{tf.path}"
+  fork do
+    `open #{tf.path}`
+  end
+  Process.wait
+  tf.unlink
+end
+
+

data/lib/mongo_aggregator.rb

@@ -0,0 +1,106 @@
+#Copyright (c) 2014, Carlo B. Bifulco. All rights reserved.
+
+require_relative "analyzer"
+require_relative "data_struct"
+require_relative "immunoscore_results_loader"
+
+
+
+class StatResults
+  include MongoMapper::Document
+  include DataUtilities
+  include CdIncluder
+  include BinManager
+  extend ClassDataUtilities
+  safe
+  timestamps!
+  key :case_n, String
+  key :file_name, String
+  key :tissue_detection_threshold, Integer
+  key :brown_threshold, Integer
+  key :mean_brown, Integer
+  key :median_brown, Float
+  key :sd_brown, Float
+  key :minimum_brown, Integer
+  key :maximum_brown, Float
+  key :mean_3_most_infiltrated_ct, Integer
+  key :density_ct_tile_1, Integer
+  key :area_ct_tile_1, Integer
+  key :number_of_cells_ct_tile_1, Integer
+  key :density_ct_tile_2, Integer
+  key :area_ct_tile_2, Integer
+  key :number_of_cells_ct_tile_2, Integer
+  key :density_ct_tile_3, Integer
+  key :area_ct_tile_3, Integer
+  key :number_of_cells_ct_tile_3, Integer
+  key :density_ct_tile_4, Integer
+  key :area_ct_tile_4, Integer
+  key :number_of_cells_ct_tile_4, Integer
+  key :density_ct_tile_5, Integer
+  key :area_ct_tile_5, Integer
+  key :number_of_cells_ct_tile_5, Integer
+  key :mean_3_most_infiltrated_im, Integer
+  key :density_im_tile_1, Integer
+  key :area_im_tile_1, Integer
+  key :number_of_cells_im_tile_1, Integer
+  key :density_im_tile_2, Integer
+  key :area_im_tile_2, Integer
+  key :number_of_cells_im_tile_2, Integer
+  key :density_im_tile_3, Integer
+  key :area_im_tile_3, Integer
+  key :number_of_cells_im_tile_3, Integer
+  key :density_im_tile_4, Integer
+  key :area_im_tile_4, Integer
+  key :number_of_cells_im_tile_4, Integer
+  key :density_im_tile_5, Integer
+  key :area_im_tile_5, Integer
+  key :number_of_cells_im_tile_5, Integer
+  key :old_mean_3_most_infiltrated_ct, Integer
+  key :old_density_ct_tile_1, Integer
+  key :old_area_ct_tile_1, Integer
+  key :old_number_of_cells_ct_tile_1, Integer
+  key :old_density_ct_tile_2, Integer
+  key :old_area_ct_tile_2, Integer
+  key :old_number_of_cells_ct_tile_2, Integer
+  key :old_density_ct_tile_3, Integer
+  key :old_area_ct_tile_3, Integer
+  key :old_number_of_cells_ct_tile_3, Integer
+  key :old_density_ct_tile_4, Integer
+  key :old_area_ct_tile_4, Integer
+  key :old_number_of_cells_ct_tile_4, Integer
+  key :old_density_ct_tile_5, Integer
+  key :old_area_ct_tile_5, Integer
+  key :old_number_of_cells_ct_tile_5, Integer
+  key :old_mean_3_most_infiltrated_im, Integer
+  key :old_density_im_tile_1, Integer
+  key :old_area_im_tile_1, Integer
+  key :old_number_of_cells_im_tile_1, Integer
+  key :old_density_im_tile_2, Integer
+  key :old_area_im_tile_2, Integer
+  key :old_number_of_cells_im_tile_2, Integer
+  key :old_density_im_tile_3, Integer
+  key :old_area_im_tile_3, Integer
+  key :old_number_of_cells_im_tile_3, Integer
+  key :old_density_im_tile_4, Integer
+  key :old_area_im_tile_4, Integer
+  key :old_number_of_cells_im_tile_4, Integer
+  key :old_density_im_tile_5, Integer
+  key :old_area_im_tile_5, Integer
+  key :old_number_of_cells_im_tile_5, Integer
+end
+
+
+### aggregates in  a single spreadsheat all csv/stat files entries present in database
+def write_stats_csv file_path
+  csv_array=[]<< Statistic.all[0].data_load.to_s.split("\n")[0]
+  ###Stat Results will contain all datasets in an csv format
+  Statistic.all.each do |stat_entry|
+    csv_array<<stat_entry.data_load.to_s.split("\n")[1]
+  end
+  csv_array
+  fh=File.new(file_path,"w")
+  csv_array.each{|l| fh.write(l+"\n")}
+  fh.close
+end
+
+