miga-base 1.2.18.1 → 1.3.0.0

data/utils/FastAAI/metadata/Accession_names_and_IDs.txt

@@ -0,0 +1,122 @@
+PF01780_19 0
+PF03948_14 1
+PF17144_4 2
+PF00830_19 3
+PF00347_23 4
+PF16906_5 5
+PF13393_6 6
+PF02565_15 7
+PF01991_18 8
+PF01984_20 9
+PF00861_22 10
+PF13656_6 11
+PF00368_18 12
+PF01142_18 13
+PF00312_22 14
+PF02367_17 15
+PF01951_16 16
+PF00749_21 17
+PF01655_18 18
+PF00318_20 19
+PF01813_17 20
+PF01649_18 21
+PF01025_19 22
+PF00380_19 23
+PF01282_19 24
+PF01864_17 25
+PF01783_23 26
+PF01808_18 27
+PF01982_16 28
+PF01715_17 29
+PF00213_18 30
+PF00119_20 31
+PF00573_22 32
+PF01981_16 33
+PF00281_19 34
+PF00584_20 35
+PF00825_18 36
+PF00406_22 37
+PF00177_21 38
+PF01192_22 39
+PF05833_11 40
+PF02699_15 41
+PF01016_19 42
+PF01765_19 43
+PF00453_18 44
+PF01193_24 45
+PF05221_17 46
+PF00231_19 47
+PF00416_22 48
+PF02033_18 49
+PF01668_18 50
+PF00886_19 51
+PF00252_18 52
+PF00572_18 53
+PF00366_20 54
+PF04104_14 55
+PF04919_12 56
+PF01912_18 57
+PF00276_20 58
+PF00203_21 59
+PF00889_19 60
+PF02996_17 61
+PF00121_18 62
+PF01990_17 63
+PF00344_20 64
+PF00297_22 65
+PF01196_19 66
+PF01194_17 67
+PF01725_16 68
+PF00750_19 69
+PF00338_22 70
+PF00238_19 71
+PF01200_18 72
+PF00162_19 73
+PF00181_23 74
+PF01866_17 75
+PF00709_21 76
+PF02006_16 77
+PF00164_25 78
+PF00237_19 79
+PF01139_17 80
+PF01351_18 81
+PF04010_13 82
+PF06093_13 83
+PF00828_19 84
+PF02410_15 85
+PF01176_19 86
+PF02130_17 87
+PF01948_18 88
+PF01195_19 89
+PF01746_21 90
+PF01667_17 91
+PF03874_16 92
+PF01090_19 93
+PF01198_19 94
+PF01250_17 95
+PF17136_4 96
+PF06026_14 97
+PF03652_15 98
+PF04019_12 99
+PF01201_22 100
+PF00832_20 101
+PF01264_21 102
+PF03840_14 103
+PF00831_23 104
+PF00189_20 105
+PF02601_15 106
+PF01496_19 107
+PF00411_19 108
+PF00334_19 109
+PF00687_21 110
+PF01157_18 111
+PF01245_20 112
+PF01994_16 113
+PF01632_19 114
+PF00827_17 115
+PF01015_18 116
+PF00829_21 117
+PF00410_19 118
+PF00833_18 119
+PF00935_19 120
+PF01992_16 121

data/utils/distance/commands.rb

@@ -146,56 +146,84 @@ module MiGA::DistanceRunner::Commands
   ##
   # Execute a FastAAI command
   def fastaai_cmd(targets)
-    qry_idx = dataset.result(:essential_genes).file_path(:fastaai_index_2)
+    qry_idx = dataset.result(:essential_genes).file_path(:fastaai_crystal)
     return nil unless qry_idx
 
     # Merge databases
     donors = []
     targets.each do |target|
-      tgt_idx = target&.result(:essential_genes)&.file_path(:fastaai_index_2)
+      tgt_idx = target&.result(:essential_genes)&.file_path(:fastaai_crystal)
       donors << tgt_idx if tgt_idx
     end
     return nil if donors.empty?
 
     # Build target database
-    f1 = tmp_file
-    if donors.size == 1
-      FileUtils.cp(donors.first, f1)
-    else
-      File.open(f0 = tmp_file, 'w') { |fh| donors.each { |i| fh.puts i } }
-      run_cmd(
-        <<~CMD
-          FastAAI merge_db --threads #{opts[:thr]} \
-            --donor_file "#{f0}" --recipient "#{f1}"
-        CMD
-      )
-      raise "Cannot merge databases into: #{f1}" unless File.size?(f1)
-    end
+    fastaai_dir = File.join(MiGA::MiGA.root_path, 'utils', 'FastAAI', 'fastaai')
+    t_db = tmp_file # Target database (from crystals)
+    q_db = tmp_file # Query database (from crystal)
+    File.open(crystals = tmp_file, 'w') { |fh| donors.each { |i| fh.puts i } }
+    script = File.join(fastaai_dir, 'fastaai_miga_crystals_to_db.py')
+    run_cmd(
+      <<~CMD
+        python3 "#{script}" \
+          --crystal_list "#{crystals}" --database_path "#{t_db}" --overwrite
+      CMD
+    )
+    raise "Cannot merge databases into: #{t_db}" unless File.size?(t_db)
+    run_cmd(
+      <<~CMD
+        echo "#{qry_idx}" | \
+          python3 "#{script}" \
+            --crystal_list /dev/stdin --database_path "#{q_db}" --overwrite
+      CMD
+    )
+    raise "Cannot merge databases into: #{q_db}" unless File.size?(q_db)
 
     # Run FastAAI
+    script = File.join(fastaai_dir, 'fastaai')
     run_cmd(
       <<~CMD
-        FastAAI db_query --query "#{qry_idx}" --target "#{f1}" \
-          --output "#{f2 = tmp_file}" --threads #{opts[:thr]} \
-          --do_stdev
+        python3 "#{script}" db_query \
+          --query "#{q_db}" --target "#{t_db}" \
+          --output "#{out_dir = tmp_file}" \
+          --threads 1 --do_stdev
       CMD
     )
-    raise "Cannot find FastAAI output directory: #{f2}" unless Dir.exist?(f2)
+    #run_cmd(
+    #  <<~CMD
+    #    python3 "#{script}" db_query --query "#{q_db}" --target "#{t_db}" \
+    #      --output "#{out_dir = tmp_file}" --threads #{opts[:thr]} \
+    #      --do_stdev
+    #  CMD
+    #)
+    raise "Cannot find FastAAI output: #{out_dir}" unless Dir.exist?(out_dir)
 
     # Save values in the databases
     haai_data = {}
     aai_data = {}
     # Ugly workaround to the insistence of FastAAI not to provide the files
     # I ask for ;-)
-    qry_results = File.basename(qry_idx, '.faix') + '_results.txt'
-    out_file = File.join(f2, 'results', qry_results)
+    # qry_results = File.basename(qry_idx, '.crystal') + '_results.txt'
+    # out_file = File.join(out_dir, 'results', qry_results)
+    out_file = Dir["#{out_dir}/results/*_results.txt"].first
+    unless out_file && File.exist?(out_file)
+      raise "Cannot find FastAAI results: #{Dir["#{out_dir}/**/*"]}"
+    end
     File.open(out_file, 'r') do |fh|
       fh.each do |ln|
         out = ln.chomp.split("\t")
         haai_data[out[1]] = [
           out[2].to_f * 100, out[3].to_f * 100, out[4].to_i, out[5].to_i
         ]
-        out[6] = (out[6] =~ /^>/) ? 95.0 : out[6].to_f
+        if out[6] =~ /^>/
+          # J-bar = 0.843 <=> AAI-hat = 90%
+          # This approximation is not in the original FastAAI paper, but it
+          # allows to maintain monotonicity at AAI-hat ≥ 90%, which solves
+          # indexing issues the ML-estimate of "AAI ~ 95%"
+          out[6] = Math.sqrt(out[2].to_f) * 100
+        else
+          out[6] = out[6].to_f
+        end
         aai_data[out[1]] = [out[6], 0, 0, 0]
       end
     end
@@ -204,7 +232,7 @@ module MiGA::DistanceRunner::Commands
     batch_data_to_db(:aai, aai_data)
 
     # Cleanup
-    [f1, f2].each { |i| FileUtils.rm_r(i) }
+    FileUtils.rm_rf([crystals, t_db, q_db, out_dir])
   end
 
   ##

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miga-base
 version: !ruby/object:Gem::Version
-  version: 1.2.18.1
+  version: 1.3.0.0
 platform: ruby
 authors:
 - Luis M. Rodriguez-R
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-03-07 00:00:00.000000000 Z
+date: 2023-03-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: daemons
@@ -287,13 +287,30 @@ files:
 - test/test_helper.rb
 - test/with_daemon_test.rb
 - test/with_option_test.rb
-- utils/FastAAI/00.Libraries/01.SCG_HMMs/Archaea_SCG.hmm
-- utils/FastAAI/00.Libraries/01.SCG_HMMs/Bacteria_SCG.hmm
-- utils/FastAAI/00.Libraries/01.SCG_HMMs/Complete_SCG_DB.hmm
-- utils/FastAAI/FastAAI
 - utils/FastAAI/FastAAI-legacy/FastAAI
 - utils/FastAAI/FastAAI-legacy/kAAI_v1.0_virus.py
+- utils/FastAAI/LICENSE
 - utils/FastAAI/README.md
+- utils/FastAAI/__init__.py
+- utils/FastAAI/example_genomes/Xanthomonas_albilineans_GCA_000962915_1.fna.gz
+- utils/FastAAI/example_genomes/Xanthomonas_albilineans_GCA_000962925_1.fna.gz
+- utils/FastAAI/example_genomes/Xanthomonas_albilineans_GCA_000962935_1.fna.gz
+- utils/FastAAI/example_genomes/Xanthomonas_albilineans_GCA_000962945_1.fna.gz
+- utils/FastAAI/example_genomes/Xanthomonas_albilineans_GCA_000962995_1.fna.gz
+- utils/FastAAI/example_genomes/Xanthomonas_albilineans_GCA_000963025_1.fna.gz
+- utils/FastAAI/example_genomes/Xanthomonas_albilineans_GCA_000963055_1.fna.gz
+- utils/FastAAI/example_genomes/Xanthomonas_albilineans_GCA_000963065_1.fna.gz
+- utils/FastAAI/example_genomes/_Pseudomonas__cissicola_GCA_002019225_1.fna.gz
+- utils/FastAAI/example_genomes/_Pseudomonas__cissicola_GCA_008801575_1.fna.gz
+- utils/FastAAI/fastaai/00.Libraries/01.SCG_HMMs/Archaea_SCG.hmm
+- utils/FastAAI/fastaai/00.Libraries/01.SCG_HMMs/Bacteria_SCG.hmm
+- utils/FastAAI/fastaai/00.Libraries/01.SCG_HMMs/Complete_SCG_DB.hmm
+- utils/FastAAI/fastaai/__init__.py
+- utils/FastAAI/fastaai/fastaai
+- utils/FastAAI/fastaai/fastaai.py
+- utils/FastAAI/fastaai/fastaai_miga_crystals_to_db.py
+- utils/FastAAI/fastaai/fastaai_miga_preproc.py
+- utils/FastAAI/metadata/Accession_names_and_IDs.txt
 - utils/adapters.fa
 - utils/cleanup-databases.rb
 - utils/core-pan-plot.R