immunoscore_results_aggregator 0.0.3

data/.gitignore

@@ -0,0 +1,4 @@
+definiens*
+*.csv
+*.R
+*.html

data/Gemfile

@@ -0,0 +1,14 @@
+source 'https://rubygems.org'
+gem "bundler"
+gem 'mongo_mapper'#, :git => "git://github.com/mongomapper/mongomapper.git", :tag => "v0.13.0.beta2"
+gem 'gibberish'
+gem 'bicrypt'
+gem 'chronic'
+gem "thor"
+gem "bson_ext"
+
+
+
+
+
+

data/Gemfile.lock

@@ -0,0 +1,47 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activemodel (4.1.6)
+      activesupport (= 4.1.6)
+      builder (~> 3.1)
+    activesupport (4.1.6)
+      i18n (~> 0.6, >= 0.6.9)
+      json (~> 1.7, >= 1.7.7)
+      minitest (~> 5.1)
+      thread_safe (~> 0.1)
+      tzinfo (~> 1.1)
+    bicrypt (1.2.0)
+    bson (1.11.1)
+    bson_ext (1.11.1)
+      bson (~> 1.11.1)
+    builder (3.2.2)
+    chronic (0.10.2)
+    gibberish (1.4.0)
+    i18n (0.6.11)
+    json (1.8.1)
+    minitest (5.4.2)
+    mongo (1.11.1)
+      bson (= 1.11.1)
+    mongo_mapper (0.13.0)
+      activemodel (>= 3.0.0)
+      activesupport (>= 3.0)
+      mongo (~> 1.8)
+      plucky (~> 0.6.5)
+    plucky (0.6.6)
+      mongo (~> 1.5)
+    thor (0.19.1)
+    thread_safe (0.3.4)
+    tzinfo (1.2.2)
+      thread_safe (~> 0.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bicrypt
+  bson_ext
+  bundler
+  chronic
+  gibberish
+  mongo_mapper
+  thor

data/README.md

@@ -0,0 +1,54 @@
+immunoscore_results_aggregator
+===========================
+
+
+- Extracts and aggregates results from the definiens immunoscore plugin to enable validation of region of interest selection and of hotspots
+- Uploads results to a nosql database (mongodb)
+
+Enables exporting of results as:
+
+- Properly structured data directory containing essential files for review 
+- As a CSV statistics summary file containing datasets for further analysis (e.g. R)
+
+
+### Mongodb datastructure
+
+####Example of immunoscore qa/qc entries: 
+
+<img src='https://notationalvelocity.s3.amazonaws.com/1-2014-09-24-11-05-11--0700.png?AWSAccessKeyId=AKIAIHNTWDGBIXEE6HEA&Signature=pXx5128pctDb72XDfTru2QcNCCU%3D&Expires=2042733911' width=' ' height=' '>  </img>
+
+
+#### Example of data structure of each entry:
+
+<img src='https://notationalvelocity.s3.amazonaws.com/3-2014-09-24-11-14-16--0700.png?AWSAccessKeyId=AKIAIHNTWDGBIXEE6HEA&Signature=DqQgv7WcgCbtjzh0g51dEUx75y8%3D&Expires=2042734457' width=' ' height=' '>  </img>
+
+Each entry contains file paths, a binary blob of the relevant snapshot, case number and relevant annotation. 
+
+
+Requirements
+===========
+     *nix
+     ruby >= 1.9.3
+     mongodb
+
+Installation
+=========
+gem install immunoscore_results_aggregator
+
+
+Setup
+=====
+
+Edit config.rb to setup the root of the immunoscore directory tree, choose a mongodb name,  and a name for the results export directory
+    
+    immunoscore_cli.rb show_config
+    immunoscore_cli.rb edit_config
+
+<img src='https://notationalvelocity.s3.amazonaws.com/4-2014-09-24-12-02-23--0700.png?AWSAccessKeyId=AKIAIHNTWDGBIXEE6HEA&Signature=R8oDkvDRMRCMZ5hgRFxNQ5pykTI%3D&Expires=2042737344' width=' ' height=' '>  </img>
+
+Run
+====
+
+    immunoscore_cli.rb
+    Follow help intructions
+

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+

data/bin/immunoscore_cli.rb

@@ -0,0 +1,76 @@
+require 'rubygems'
+require 'bundler/setup'
+require "yaml"
+require_relative "../immunoscore_results_aggregator"
+#!/usr/bin/env ruby
+# coding: utf-8
+
+
+require "thor"
+require "date"
+
+
+
+def check_config
+  raise "cannot find definiens base directory.  please check and edit configuration file"    unless Dir.exists? BASE_DIR
+  Dir.mkdir config["export_dir"] unless Dir.exists? EXPORT_DIR
+end
+
+
+class CliSummary < Thor                                                 # [1]
+  package_name "Cli Summary"                                             # [2]
+  #map "-L" => :list                                              # [3]
+
+  # desc "install APP_NAME", "install one of the available apps"   # [4]
+  # method_options :force => :boolean, :alias => :string           # [5]
+  # def install(name)
+  #   user_alias = options[:alias]
+  #   if options.force?
+  #     # do something
+  #   end
+  #   # other code
+  # end
+
+  # desc "list [SEARCH]", "list all of the available apps, limited by SEARCH"
+  # def list(search="")
+  #   # list everything
+  # end
+  desc "export_summary_csv [N]", "merges and exports all stat results; export filename as an argument "
+  def export_summary_csv all_results_file_name="all_definiens_results_#{Date.today.to_s}.csv"
+    write_stats_csv all_results_file_name
+    puts "\n\n\n"
+    puts "Results are available in #{all_results_file_name}"
+    puts "\n\n\n"
+
+  end
+
+  desc "upload_data", "uploads definines immunoscore directory into mongo; no arguments"
+  def upload_data
+    check_config
+    mongo_load_all
+  end
+
+  desc "export_data_struct", "exports data structure of all uploaded datasets; no arguments"
+  def export_data_struct
+    check_config
+    export_mongo
+  end
+
+  desc "show_config", "shows config filepath; no arguments"
+  def show_config
+    ShowConfig.show_config
+  end
+  desc "edit config", "edit config file; no argumens"
+  def edit_config
+    begin 
+      `emacs #{ShowConfig.show_config}`
+    rescue
+    end
+  end
+
+
+
+end
+
+
+CliSummary.start(ARGV)

data/config.rb

@@ -0,0 +1,14 @@
+# export destination directory for reformatted immunoscore results
+EXPORT_DIR= "/Users/carlobifulco/Desktop/fda_amp"
+
+# source immunoscore directory
+BASE_DIR= "/Users/carlobifulco/Desktop/fda_amp"
+# data base name
+DATABASE_NAME= "immunoscore_results_aggregator"
+
+module ShowConfig
+  def self.show_config
+    puts "PATH: #{File.absolute_path(__FILE__)}"
+    File.absolute_path(__FILE__)
+    end
+end

data/data_struct.rb

@@ -0,0 +1,280 @@
+#Copyright (c) 2014, Carlo B. Bifulco. All rights reserved.
+
+require_relative "analyzer"
+require_relative "immunoscore_results_loader"
+
+require_relative "database_connection"
+
+# module Show
+#   def show file_path=self.path
+#     tf=Tempfile.new ["temp",File.extname(file_path)]
+#     tf.write (File.read file_path)
+#     tf.close
+#     puts "#{tf.path}"
+#     fork do
+#       `open #{tf.path}`
+
+#     end
+#     puts Process.waitall
+#     tf.unlink
+#   end
+# end
+
+### Enables all Mongo classes listed below to have instances capable their own CD quicly
+module CdIncluder
+  def get_cd
+    puts self.cd_type
+    if (self.cd_type and self.case_n) != nil 
+      cd= Cd.get(self.cd_type, self.case_n)
+    end
+    puts cd
+    if cd
+      self.cd=cd
+    end
+    self.save
+  end
+end
+
+### enables loading and showing of binary datatsets
+module BinManager
+  def load_bin
+    self.data_load=File.read path
+  end
+  
+  def show graphic_data=self.data_load
+    tf=Tempfile.new ["temp",".jpg"]
+    tf.write graphic_data
+    tf.close
+    puts "#{tf.path}"
+    fork do
+      `open #{tf.path}`
+    end
+    Process.wait
+    tf.unlink
+  end
+end
+
+class ImmunoScoreResults
+  include MongoMapper::Document
+  include DataUtilities
+  safe
+  timestamps!
+  many :cd
+  key :case_n, String
+
+  def self.get case_n
+    self.find_by_case_n(case_n) or self.create(:case_n=>case_n)
+  end
+
+end
+
+
+class Cd
+  include MongoMapper::Document
+  include DataUtilities
+  safe
+  timestamps!
+  belongs_to :immuno_score_results
+  key :cd_type, String
+  key :case_n, String
+  many :ct_tile
+  many :im_tile
+  many :statistic
+  many :classification
+  many :original
+  many :density
+  many :histogram
+
+  #either get a brand new cd with its own Immunoscore Results instance of find a preexisting one
+  def self.get cd_type, case_n
+    self.find_by_cd_type_and_case_n(cd_type,case_n) or self.create(:case_n=>case_n, 
+                                                                      :cd_type=>cd_type,
+                                                                      :immuno_score_results_id=>ImmunoScoreResults.get(case_n)._id)
+  end
+end
+
+
+
+
+  # {:cd=>"CD3",
+  # :path=>
+  #  "/Volumes/LaCie/patient 1-5/patient 1-5/results/RS_SM02-2576-A4_CD3_2013-11-01 16_52_41.image_0000006553/View_Exports
+  # :tile=>"CT2",
+  # :case_n=>"SM02-2576-A4",
+  # :type=>:ct_tile}}
+
+class CtTile
+  include MongoMapper::Document
+  include DataUtilities
+  include CdIncluder
+  include BinManager
+  extend ClassDataUtilities
+  safe
+  timestamps!
+  belongs_to :cd
+  key :tile, String
+  key :path, String
+  key :case_n, String
+  key :cd_type, String
+  key :data_load, Binary
+  before_save :load_bin
+end
+
+
+
+class ImTile
+  include MongoMapper::Document
+  include DataUtilities
+  include CdIncluder
+  include BinManager
+  extend ClassDataUtilities
+  safe
+  timestamps!
+  belongs_to :cd
+  key :tile, String
+  key :path, String
+  key :case_n, String
+  key :cd_type, String
+  key :data_load, Binary
+  before_save :load_bin
+end
+
+# {:case=>"SM02-2456-A6",
+#   :cd=>"CD8",
+#   :path=>
+#    "/Volumes/LaCie/patient 1-5/patient 1-5/results/RS_SM02-2456-A6_CD8_2013-11-01 14_27_27.image_0000006515/image_000000
+#   :type=>:classification},
+class Classification
+  include MongoMapper::Document
+  include DataUtilities
+  include CdIncluder
+  include BinManager
+  extend ClassDataUtilities
+  safe
+  timestamps!
+  belongs_to :cd
+  key :path, String
+  key :case_n, String
+  key :cd_type, String
+  key :data_load, Binary
+  before_save :load_bin
+end
+
+class Original
+  include MongoMapper::Document
+  include DataUtilities
+  include CdIncluder
+  include BinManager
+  extend ClassDataUtilities
+  safe
+  timestamps!
+  belongs_to :cd
+  key :path, String
+  key :case_n, String
+  key :cd_type, String
+  key :data_load, Binary
+  before_save :load_bin
+end
+
+ # {:case=>"RS_SM02-2576-A4_CD3_2013-11-01 16_52_41.image_0000006553",
+ #  :path=>
+ #   "/Volumes/LaCie/patient 1-5/patient 1-5/results/RS_SM02-2576-A4_CD3_2013-11-01 16_52_41.image_0000006553/Statistic_Exports/RS_SM02-2576-A4_CD3_2013-11-01 16_52_41.image_0000006553_Statistics.csv",
+ #  :cd=>"CD3",
+ #  :type=>:statistics}
+
+class Statistic
+  include MongoMapper::Document
+  include DataUtilities
+  include CdIncluder
+  include BinManager
+  extend ClassDataUtilities
+  safe
+  timestamps!
+  belongs_to :cd
+  key :path, String
+  key :case_n, String
+  key :cd_type, String
+  key :data_load, Binary
+  before_save :load_bin
+end
+
+
+ # {:path=>
+ #   "/Volumes/LaCie/patient 1-5/patient 1-5/results/RS_SM02-2576-A4_CD3_2013-11-01 16_52_41.image_0000006553/View_Exports
+ #  :case=>"SM02-2576-A4",
+ #  :new_old=>"old",
+ #  :cd=>"CD3",
+ #  :type=>"density"}
+class Density
+  include MongoMapper::Document
+  include DataUtilities
+  include CdIncluder
+  include BinManager
+  extend ClassDataUtilities
+  safe
+  timestamps!
+  belongs_to :cd
+  key :path, String
+  key :case_n, String
+  key :cd_type, String
+  key :new_old, String
+  key :data_load, Binary
+  before_save :load_bin
+end
+
+
+ # {:path=>
+ #   "/Volumes/LaCie/patient 1-5/patient 1-5/results/RS_SM02-2366-A5_CD8_2013-11-01 16_55_57.image_0000006556/View_Exports
+ #  :case=>"SM02-2366-A5",
+ #  :type=>:histogram,
+ #  :cd=>"CD8"}
+class Histogram
+  include MongoMapper::Document
+  include DataUtilities
+  include CdIncluder
+  include BinManager
+  extend ClassDataUtilities
+  safe
+  timestamps!
+  belongs_to :cd
+  key :path, String
+  key :case_n, String
+  key :cd_type, String
+  key :data_load, Binary
+  before_save :load_bin
+end
+
+
+class Test
+  include MongoMapper::Document
+  include DataUtilities
+  include CdIncluder
+  include BinManager
+  extend ClassDataUtilities
+ 
+  key :path, String
+  
+  key :data_load, Binary
+  before_save :load_bin
+
+
+end
+
+
+
+### mapping from definiens file type to proper database entity
+JSON_CLASS_MAPPER={:ct_tile=>CtTile,
+  :im_tile=>ImTile,
+  :classification=>Classification,
+  :original=>Original,
+  :statistic=>Statistic,
+  :density=>Density,
+  :histogram=>Histogram}
+
+
+### resets all databases
+def mm_clean_all
+  [ImmunoScoreResults, Cd, Histogram, Density, Statistic, Original, Classification, ImTile, CtTile].each do |mm_class|
+    mm_class.delete_all
+  end
+end