sequenceserver 2.0.0.rc8 → 2.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (116) hide show
  1. checksums.yaml +4 -4
  2. data/bin/sequenceserver +22 -30
  3. data/lib/sequenceserver/api_errors.rb +5 -1
  4. data/lib/sequenceserver/blast/constants.rb +1 -1
  5. data/lib/sequenceserver/blast/hit.rb +5 -16
  6. data/lib/sequenceserver/blast/job.rb +9 -18
  7. data/lib/sequenceserver/blast/report.rb +5 -3
  8. data/lib/sequenceserver/config.rb +4 -1
  9. data/lib/sequenceserver/database.rb +69 -9
  10. data/lib/sequenceserver/job.rb +1 -1
  11. data/lib/sequenceserver/makeblastdb.rb +40 -45
  12. data/lib/sequenceserver/routes.rb +4 -0
  13. data/lib/sequenceserver/version.rb +1 -1
  14. data/lib/sequenceserver.rb +15 -11
  15. data/public/config.js +143 -142
  16. data/public/css/fonts.css +23 -22
  17. data/public/css/grapher.css +598 -594
  18. data/public/css/sequenceserver.css +86 -24
  19. data/public/css/sequenceserver.min.css +2 -2
  20. data/public/js/alignment_exporter.js +14 -14
  21. data/public/js/databases_tree.js +215 -0
  22. data/public/js/download_fasta.js +1 -1
  23. data/public/js/hit.js +6 -2
  24. data/public/js/hits_overview.js +1 -1
  25. data/public/js/length_distribution.js +5 -5
  26. data/public/js/query.js +4 -7
  27. data/public/js/report.js +12 -24
  28. data/public/js/search.js +21 -2
  29. data/public/js/sidebar.js +4 -4
  30. data/public/js/svgExporter.js +12 -12
  31. data/public/js/visualisation_helpers.js +4 -5
  32. data/public/sequenceserver-report.min.js +11 -11
  33. data/public/sequenceserver-search.min.js +15 -11
  34. data/public/vendor/github/vakata/jstree@3.3.8/LICENSE-MIT +22 -0
  35. data/public/vendor/github/vakata/jstree@3.3.8/README.md +663 -0
  36. data/public/vendor/github/vakata/jstree@3.3.8/bower.json +33 -0
  37. data/public/vendor/github/vakata/jstree@3.3.8/component.json +28 -0
  38. data/public/vendor/github/vakata/jstree@3.3.8/composer.json +46 -0
  39. data/public/vendor/github/vakata/jstree@3.3.8/demo/README.md +2 -0
  40. data/public/vendor/github/vakata/jstree@3.3.8/demo/basic/index.html +146 -0
  41. data/public/vendor/github/vakata/jstree@3.3.8/demo/basic/root.json +1 -0
  42. data/public/vendor/github/vakata/jstree@3.3.8/dist/jstree.js +8612 -0
  43. data/public/vendor/github/vakata/jstree@3.3.8/dist/jstree.min.js +6 -0
  44. data/public/vendor/github/vakata/jstree@3.3.8/dist/themes/default/32px.png +0 -0
  45. data/public/vendor/github/vakata/jstree@3.3.8/dist/themes/default/40px.png +0 -0
  46. data/public/vendor/github/vakata/jstree@3.3.8/dist/themes/default/style.css +1102 -0
  47. data/public/vendor/github/vakata/jstree@3.3.8/dist/themes/default/style.min.css +1 -0
  48. data/public/vendor/github/vakata/jstree@3.3.8/dist/themes/default/throbber.gif +0 -0
  49. data/public/vendor/github/vakata/jstree@3.3.8/dist/themes/default-dark/32px.png +0 -0
  50. data/public/vendor/github/vakata/jstree@3.3.8/dist/themes/default-dark/40px.png +0 -0
  51. data/public/vendor/github/vakata/jstree@3.3.8/dist/themes/default-dark/style.css +1146 -0
  52. data/public/vendor/github/vakata/jstree@3.3.8/dist/themes/default-dark/style.min.css +1 -0
  53. data/public/vendor/github/vakata/jstree@3.3.8/dist/themes/default-dark/throbber.gif +0 -0
  54. data/public/vendor/github/vakata/jstree@3.3.8/gruntfile.js +242 -0
  55. data/public/vendor/github/vakata/jstree@3.3.8/jstree.jquery.json +28 -0
  56. data/public/vendor/github/vakata/jstree@3.3.8/package.json +58 -0
  57. data/public/vendor/github/vakata/jstree@3.3.8/src/intro.js +14 -0
  58. data/public/vendor/github/vakata/jstree@3.3.8/src/jstree.changed.js +69 -0
  59. data/public/vendor/github/vakata/jstree@3.3.8/src/jstree.checkbox.js +976 -0
  60. data/public/vendor/github/vakata/jstree@3.3.8/src/jstree.conditionalselect.js +38 -0
  61. data/public/vendor/github/vakata/jstree@3.3.8/src/jstree.contextmenu.js +661 -0
  62. data/public/vendor/github/vakata/jstree@3.3.8/src/jstree.dnd.js +669 -0
  63. data/public/vendor/github/vakata/jstree@3.3.8/src/jstree.js +4931 -0
  64. data/public/vendor/github/vakata/jstree@3.3.8/src/jstree.massload.js +137 -0
  65. data/public/vendor/github/vakata/jstree@3.3.8/src/jstree.search.js +421 -0
  66. data/public/vendor/github/vakata/jstree@3.3.8/src/jstree.sort.js +74 -0
  67. data/public/vendor/github/vakata/jstree@3.3.8/src/jstree.state.js +138 -0
  68. data/public/vendor/github/vakata/jstree@3.3.8/src/jstree.types.js +372 -0
  69. data/public/vendor/github/vakata/jstree@3.3.8/src/jstree.unique.js +164 -0
  70. data/public/vendor/github/vakata/jstree@3.3.8/src/jstree.wholerow.js +122 -0
  71. data/public/vendor/github/vakata/jstree@3.3.8/src/misc.js +656 -0
  72. data/public/vendor/github/vakata/jstree@3.3.8/src/outro.js +1 -0
  73. data/public/vendor/github/vakata/jstree@3.3.8/src/sample.js +93 -0
  74. data/public/vendor/github/vakata/jstree@3.3.8/src/themes/base.less +93 -0
  75. data/public/vendor/github/vakata/jstree@3.3.8/src/themes/default/32px.png +0 -0
  76. data/public/vendor/github/vakata/jstree@3.3.8/src/themes/default/40px.png +0 -0
  77. data/public/vendor/github/vakata/jstree@3.3.8/src/themes/default/style.css +1102 -0
  78. data/public/vendor/github/vakata/jstree@3.3.8/src/themes/default/style.less +22 -0
  79. data/public/vendor/github/vakata/jstree@3.3.8/src/themes/default/throbber.gif +0 -0
  80. data/public/vendor/github/vakata/jstree@3.3.8/src/themes/default-dark/32px.png +0 -0
  81. data/public/vendor/github/vakata/jstree@3.3.8/src/themes/default-dark/40px.png +0 -0
  82. data/public/vendor/github/vakata/jstree@3.3.8/src/themes/default-dark/style.css +1146 -0
  83. data/public/vendor/github/vakata/jstree@3.3.8/src/themes/default-dark/style.less +50 -0
  84. data/public/vendor/github/vakata/jstree@3.3.8/src/themes/default-dark/throbber.gif +0 -0
  85. data/public/vendor/github/vakata/jstree@3.3.8/src/themes/main.less +77 -0
  86. data/public/vendor/github/vakata/jstree@3.3.8/src/themes/mixins.less +104 -0
  87. data/public/vendor/github/vakata/jstree@3.3.8/src/themes/responsive.less +67 -0
  88. data/public/vendor/github/vakata/jstree@3.3.8/src/vakata-jstree.js +38 -0
  89. data/public/vendor/github/vakata/jstree@3.3.8/test/unit/index.html +16 -0
  90. data/public/vendor/github/vakata/jstree@3.3.8/test/unit/libs/qunit.css +244 -0
  91. data/public/vendor/github/vakata/jstree@3.3.8/test/unit/libs/qunit.js +2212 -0
  92. data/public/vendor/github/vakata/jstree@3.3.8/test/unit/test.js +11 -0
  93. data/public/vendor/github/vakata/jstree@3.3.8/test/visual/desktop/index.html +44 -0
  94. data/public/vendor/github/vakata/jstree@3.3.8/test/visual/mobile/index.html +42 -0
  95. data/public/vendor/github/vakata/jstree@3.3.8/test/visual/screenshots/desktop/desktop.png +0 -0
  96. data/public/vendor/github/vakata/jstree@3.3.8/test/visual/screenshots/desktop/home.png +0 -0
  97. data/public/vendor/github/vakata/jstree@3.3.8/test/visual/screenshots/mobile/home.png +0 -0
  98. data/public/vendor/github/vakata/jstree@3.3.8/test/visual/screenshots/mobile/mobile.png +0 -0
  99. data/public/vendor/github/vakata/jstree@3.3.8.js +3 -0
  100. data/public/vendor/system-csp-production.js +3 -3
  101. data/public/vendor/system-csp-production.js.map +1 -1
  102. data/public/vendor/system-csp-production.src.js +146 -140
  103. data/public/vendor/system-polyfills.js.map +1 -1
  104. data/public/vendor/system-polyfills.src.js +1 -0
  105. data/public/vendor/system.js +3 -3
  106. data/public/vendor/system.js.map +1 -1
  107. data/public/vendor/system.src.js +4771 -2383
  108. data/views/_options.erb +21 -0
  109. data/views/layout.erb +17 -18
  110. metadata +102 -43
  111. data/bin/chromedriver +0 -0
  112. data/bin/geckodriver +0 -0
  113. data/public/shims/form-core.js +0 -3
  114. data/public/shims/form-validation.js +0 -3
  115. data/public/shims/plugins/jquery.ui.position.js +0 -13
  116. data/public/shims/styles/shim.css +0 -1
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 449efaa2619fd2c83a4116480c78ae6b892804f2f9bc23c3d074b8b0493985c5
4
- data.tar.gz: 393f9da8bd3df53bff3ed3a137a5d9484ad5ec9cfa1de3c7b15c3f1a6833f092
3
+ metadata.gz: 25c2424857780517682276d21694f8ab0c3594772b82f08dcdca1934dcf23fd9
4
+ data.tar.gz: 3e27329d829825b7a3a73f14ebdecad514648381dbba914cfca6ee63564c795a
5
5
  SHA512:
6
- metadata.gz: de2fdcef61a8fcdbe8907935a1f0b545219698fe9d37fcad6cd40aaa50e779010b7e91a9158eb76590baed78ce3743cdc173b664ff5d0969dfaca42eb6bd621c
7
- data.tar.gz: a65c243d19435021b00a360ddfa8f617fa58e5944dd55117a8865f11dec4554a86e577cdfee3013a82638dc45de697a33156da1ba0a2fedd8e2936a7d760d5d9
6
+ metadata.gz: 5137bf6118ee0044a0b8d5b1979f525686d7dd1698d6c795682d39a6c9d8247410104441fb642a585e6df9a18ad827336c8f61215f9e3c8c68fb387e0b6e2f41
7
+ data.tar.gz: e7ec5cebd2595b3a5f90cb750ce583161821e1be22dfcf14b1ba1fbe11dda440105795ac04b06caeb456c0f78e55db50d730704db8972667ca8d7acae5170692
data/bin/sequenceserver CHANGED
@@ -308,34 +308,6 @@ begin
308
308
  exit! unless set?
309
309
  end
310
310
  end
311
- rescue SequenceServer::INCOMPATIBLE_BLAST_DATABASES => e
312
- unless list_databases? || make_blast_databases?
313
- # Print error raised.
314
- puts
315
- puts e
316
-
317
- # Offer user to format the FASTA files.
318
- database_dir = SequenceServer.config[:database_dir]
319
- puts
320
- puts <<~MSG
321
- Incompatible databases can cause BLAST searches and other features of
322
- SequenceServer to fail unexpectedly.
323
- You can view incompatible databases and choose to reformat them below.
324
- Alternatively, please remove them from databases directory.
325
-
326
- View incompatible databases? [y/n] (Default: y).
327
- MSG
328
- puts
329
- print '>> '
330
- response = STDIN.gets.to_s.strip
331
- unless response =~ /^[n]$/i
332
- reformatted = SequenceServer.makeblastdb.reformat
333
- exit! if reformatted.empty? && !set?
334
- redo unless set?
335
- else
336
- exit! unless set?
337
- end
338
- end
339
311
  rescue SequenceServer::ENOENT,
340
312
  SequenceServer::CONFIG_FILE_ERROR,
341
313
  SequenceServer::BLAST_DATABASE_ERROR,
@@ -382,8 +354,28 @@ begin
382
354
 
383
355
  if make_blast_databases?
384
356
  if SequenceServer.makeblastdb.scan
385
- SequenceServer.makeblastdb.run
386
- else
357
+ puts
358
+ puts <<~MSG
359
+ SequenceServer has scanned your databases directory and will now offer
360
+ to convert FASTA files into BLAST databases. It will also offer to
361
+ reformat any old-format BLAST databases and those created without
362
+ the -parse_seqids option of makeblastdb (-parse_seqids option is
363
+ required for sequence retrieval to correctly work).
364
+
365
+ Note that reformatting process can be slow large BLAST databases and
366
+ fail if sequence identifiers are longer than 50 characters. While we
367
+ exepect the reformatting process to work in most other cases, things
368
+ can inevitably go wrong. Thus, please back up your databases before
369
+ reformatting and post any issues to our Google Group/GitHub so that
370
+ we can all learn from it.
371
+
372
+ Proceed? [y/n] (Default: y).
373
+ MSG
374
+ puts
375
+ print '>> '
376
+ response = STDIN.gets.to_s.strip
377
+ SequenceServer.makeblastdb.run unless response =~ /^[n]$/i
378
+ else
387
379
  puts "All FASTA files in #{SequenceServer.config[:database_dir]} " \
388
380
  'are formatted.'
389
381
  end
@@ -41,7 +41,11 @@ module SequenceServer
41
41
  def message
42
42
  <<~MSG
43
43
  Looks like there's a problem with one of the query sequences, selected
44
- databases, or advanced parameters.
44
+ databases, or advanced parameters. Details of the error are included
45
+ below. Please ask on our
46
+ <a href="https://github.com/wurmlab/sequenceserver/issues" target="_blank">issue tracker</a>
47
+ or on our <a href="https://groups.google.com/g/sequenceserver">forum</a> if you are
48
+ not sure what the error message means, or if the error message is just a number.
45
49
  MSG
46
50
  end
47
51
 
@@ -2,7 +2,7 @@
2
2
  module SequenceServer
3
3
  # Define constanst used by BLAST module.
4
4
  module BLAST
5
- ERROR_LINE = /Error:\s(.*)/
5
+ ERROR_LINE = /Error:\s(.*)/i
6
6
 
7
7
  ALGORITHMS = %w[blastn blastp blastx tblastn tblastx].freeze
8
8
 
@@ -21,8 +21,8 @@ module SequenceServer
21
21
  # representation.
22
22
  def to_json(*args)
23
23
  # List all attributes that we want to send to the browser.
24
- properties = %i[number id accession title length score identity
25
- qcovs sciname evalue hsps links]
24
+ properties = %i[number id accession title length total_score
25
+ qcovs sciname hsps links]
26
26
  properties.inject({}) { |h, k| h[k] = send(k); h }.to_json(*args)
27
27
  end
28
28
 
@@ -71,23 +71,12 @@ module SequenceServer
71
71
  # client.
72
72
  ###
73
73
 
74
- # Returns the sum of scores of all HSPs.
75
- def score
74
+ # Returns the sum of scores of all HSPs. Displayed in the tabular summary
75
+ # of hits in the HTML report. Should probably be calculated in browser?
76
+ def total_score
76
77
  hsps.map(&:score).reduce(:+)
77
78
  end
78
79
 
79
- # Returns the sum of identity of all HSPs divided by sum of length of all
80
- # HSPs (expressed as percentage).
81
- def identity
82
- hsps.map(&:identity).reduce(:+) * 100 / hsps.map(&:length).reduce(:+)
83
- end
84
-
85
- # Returns the minimum evalue of all HSPs of the Hit. This is shown in the
86
- # tabular overview of hits in the HTML report.
87
- def evalue
88
- hsps.first.evalue
89
- end
90
-
91
80
  private
92
81
 
93
82
  # Returns the report object that this hit is a part of. This is used to
@@ -60,28 +60,19 @@ module SequenceServer
60
60
 
61
61
  # Handle error. See [1].
62
62
  case exitstatus
63
- when 1
64
- # Error in query sequence or options.
63
+ when 1..2
64
+ # 1: Error in query sequences or options.
65
+ # 2: Error in BLAST databases.
65
66
  error = IO.foreach(stderr).grep(ERROR_LINE).join
66
67
  error = File.read(stderr) if error.empty?
67
- fail InputError, error
68
- when 2
69
- fail InputError, <<~MSG
70
- BLAST signalled a problem with the databases that you searched.
71
-
72
- Most likely one or more of your databases were created using an
73
- older version of BLAST. Please consider recreating the databases
74
- using BLAST #{BLAST_VERSION}.
75
-
76
- As a temporary solution, you can try searching one database at a time.
77
- MSG
68
+ fail InputError, "(#{exitstatus}) #{error}"
78
69
  when 4
79
70
  # Out of memory. User can retry with a shorter search, so raising
80
71
  # InputError here instead of SystemError.
81
72
  fail InputError, <<~MSG
82
- Ran out of memory. Please try a smaller query, or searching fewer and smaller
73
+ Ran out of memory. Please try a smaller query, fewer and smaller
83
74
  databases, or limiting the output by using advanced options.
84
- MSG
75
+ MSG
85
76
  when 6
86
77
  # Error creating output files. It can't be a permission issue as that
87
78
  # would have been caught while creating job directory. But we can run
@@ -98,7 +89,7 @@ module SequenceServer
98
89
  fail SystemError, <<~MSG
99
90
  BLAST failed abruptly (exit status: #{exitstatus}). Most likely there is a
100
91
  problem with the BLAST+ binaries.
101
- MSG
92
+ MSG
102
93
  end
103
94
  end
104
95
  # rubocop:enable Metrics/CyclomaticComplexity
@@ -155,7 +146,7 @@ module SequenceServer
155
146
  end
156
147
 
157
148
  def allowed_chars
158
- /\A[a-z0-9\-_\. ']*\Z/i
149
+ /\A[a-z0-9\-_\. ',]*\Z/i
159
150
  end
160
151
 
161
152
  def disallowed_options
@@ -167,4 +158,4 @@ end
167
158
 
168
159
  # References
169
160
  # ----------
170
- # [1]: http://www.ncbi.nlm.nih.gov/books/NBK1763/
161
+ # [1]: http://www.ncbi.nlm.nih.gov/books/NBK1763/ (Appendices)
@@ -25,6 +25,7 @@ module SequenceServer
25
25
  def initialize(job)
26
26
  super do
27
27
  @queries = []
28
+ @querydb = job.databases
28
29
  end
29
30
  end
30
31
 
@@ -42,7 +43,8 @@ module SequenceServer
42
43
  }.update(search_id: job.id,
43
44
  submitted_at: job.submitted_at.utc,
44
45
  imported_xml: !!job.imported_xml_file,
45
- seqserv_version: SequenceServer::VERSION).to_json
46
+ seqserv_version: SequenceServer::VERSION,
47
+ non_parse_seqids: !!job.databases&.any?(&:non_parse_seqids?)).to_json
46
48
  end
47
49
 
48
50
  private
@@ -80,13 +82,12 @@ module SequenceServer
80
82
  # Get database information (title and type) from job yaml or from XML.
81
83
  # Sets `querydb` and `dbtype` attributes.
82
84
  def extract_db_info(ir)
83
- if job.databases.empty?
85
+ if @querydb.empty?
84
86
  @querydb = ir[3].split.map do |path|
85
87
  { title: File.basename(path) }
86
88
  end
87
89
  @dbtype = dbtype_from_program
88
90
  else
89
- @querydb = job.databases
90
91
  @dbtype = @querydb.first.type
91
92
  end
92
93
  end
@@ -176,6 +177,7 @@ module SequenceServer
176
177
  def parse_xml(xml)
177
178
  node_to_array Ox.parse(xml).root
178
179
  rescue Ox::ParseError
180
+ fail 'Error parsing XML file' if job.imported_xml_file
179
181
  fail InputError, <<~MSG
180
182
  BLAST generated incorrect XML output. This can happen if sequence ids in your
181
183
  databases are not unique across all files. As a temporary workaround, you can
@@ -99,6 +99,7 @@ module SequenceServer
99
99
  {
100
100
  host: '0.0.0.0',
101
101
  port: 4567,
102
+ databases_widget: 'classic',
102
103
  options: {
103
104
  blastn: ['-task blastn', '-evalue 1e-5'],
104
105
  blastp: ['-evalue 1e-5'],
@@ -106,7 +107,9 @@ module SequenceServer
106
107
  tblastx: ['-evalue 1e-5'],
107
108
  tblastn: ['-evalue 1e-5']
108
109
  },
109
- num_threads: 1
110
+ num_threads: 1,
111
+ num_jobs: 1,
112
+ job_lifetime: 43200
110
113
  }
111
114
  end
112
115
  end
@@ -19,14 +19,14 @@ module SequenceServer
19
19
  # SequenceServer will always place BLAST database files alongside input FASTA,
20
20
  # and use `parse_seqids` option of `makeblastdb` to format databases.
21
21
  Database = Struct.new(:name, :title, :type, :nsequences, :ncharacters,
22
- :updated_on) do
22
+ :updated_on, :format, :categories) do
23
23
 
24
24
  extend Forwardable
25
25
 
26
26
  def_delegators SequenceServer, :config, :sys
27
27
 
28
28
  def initialize(*args)
29
- args[2].downcase! # database type
29
+ args[2].downcase! # type
30
30
  args.each(&:freeze)
31
31
  super
32
32
 
@@ -34,6 +34,7 @@ module SequenceServer
34
34
  end
35
35
 
36
36
  attr_reader :id
37
+ alias path name
37
38
 
38
39
  def retrieve(accession, coords = nil)
39
40
  cmd = "blastdbcmd -db #{name} -entry '#{accession}'"
@@ -48,10 +49,36 @@ module SequenceServer
48
49
  nil
49
50
  end
50
51
 
51
- def include?(accession)
52
- cmd = "blastdbcmd -entry '#{accession}' -db #{name}"
53
- out, = sys(cmd, path: config[:bin])
54
- !out.empty?
52
+ # Returns true if the database contains the given sequence id.
53
+ # Returns false otherwise.
54
+ def include?(id)
55
+ cmd = "blastdbcmd -entry '#{id}' -db #{name}"
56
+ sys(cmd, path: config[:bin]) rescue false
57
+ end
58
+
59
+ def v4?
60
+ format == '4'
61
+ end
62
+
63
+ def v5?
64
+ format == '5'
65
+ end
66
+
67
+ # Return true if the database was _not_ created using the -parse_seqids
68
+ # option of makeblastdb.
69
+ def non_parse_seqids?
70
+ return if alias?
71
+ case format
72
+ when '5'
73
+ (%w[nog nos pog pos] & extensions).length != 2
74
+ when '4'
75
+ (%w[nog nsd nsi pod psd psi] & extensions).length != 3
76
+ end
77
+ end
78
+
79
+ # Returns true if the database was created using blastdb_aliastool.
80
+ def alias?
81
+ (%w[nal pal] & extensions).length == 1 && extensions.count == 1
55
82
  end
56
83
 
57
84
  def ==(other)
@@ -65,6 +92,16 @@ module SequenceServer
65
92
  def to_json(*args)
66
93
  to_h.update(id: id).to_json(*args)
67
94
  end
95
+
96
+ private
97
+
98
+ def extensions
99
+ # The glob pattern used here is quite relaxed. This is to capture
100
+ # multipart databases as well. It is possible that non-blast-database
101
+ # extensions may also be picked. However, that shouldn't be a problem
102
+ # as we only check whether certain required extensions are present or not.
103
+ @extensions ||= Dir["#{path}*{n,p}*"].map { |p| p.split('.').last }.sort.uniq
104
+ end
68
105
  end
69
106
 
70
107
  # Model Database's eigenclass as a collection of Database objects.
@@ -80,9 +117,8 @@ module SequenceServer
80
117
  @collection ||= {}
81
118
  end
82
119
 
83
- def collection=(databases_attrs)
84
- databases_attrs.each do |db_attrs|
85
- db = Database.new(*db_attrs)
120
+ def collection=(databases)
121
+ databases.each do |db|
86
122
  collection[db.id] = db
87
123
  end
88
124
  end
@@ -102,6 +138,30 @@ module SequenceServer
102
138
  collection.values
103
139
  end
104
140
 
141
+ def tree
142
+ all.each_with_object({}) do |db, data|
143
+ data[db.type] ||= []
144
+ use_parent = '#'
145
+ db.categories.each_with_index do |entry, index|
146
+ parent = index.zero? ? '#' : db.categories[0..(index - 1)].join('-')
147
+ use_id = db.categories[0..index].join('-')
148
+ element = { id: use_id, parent: parent, text: entry }
149
+ data[db.type] << element unless data[db.type].include?(element)
150
+ use_parent = use_id
151
+ end
152
+
153
+ data[db.type] <<
154
+ {
155
+ id: db.id,
156
+ parent: use_parent,
157
+ text: db.title,
158
+ icon: 'glyphicon glyphicon-file'
159
+ }
160
+
161
+ yield(db, data[db.type].last) if block_given?
162
+ end
163
+ end
164
+
105
165
  def each(&block)
106
166
  all.each(&block)
107
167
  end
@@ -51,7 +51,7 @@ module SequenceServer
51
51
 
52
52
  # Thread pool used for running BLAST searches.
53
53
  def pool
54
- @pool ||= Pool.new(SequenceServer.config[:num_threads])
54
+ @pool ||= Pool.new(SequenceServer.config[:num_jobs] || 1)
55
55
  end
56
56
  end
57
57
 
@@ -11,17 +11,6 @@ module SequenceServer
11
11
  # makeblastdb.scan && makeblastdb.run
12
12
  #
13
13
  class MAKEBLASTDB
14
- # We want V5 databases created using -parse_seqids for proper function of
15
- # SequenceServer. This means each database should be comprised of at least 9
16
- # files with the following extensions. Databases created by us will have two
17
- # additional files with the extensions nhd and nhi, or phd and phi, due to
18
- # the use of -hash_index option. Finally, multipart databases will have one
19
- # additional file with the extension nal or pal.
20
- REQUIRED_EXTENSIONS = {
21
- 'nucleotide' => %w{ndb nhr nin nog nos not nsq ntf nto}.freeze,
22
- 'protein' => %w{pdb phr pin pog pos pot psq ptf pto}.freeze
23
- }
24
-
25
14
  extend Forwardable
26
15
 
27
16
  def_delegators SequenceServer, :config, :sys
@@ -70,18 +59,12 @@ module SequenceServer
70
59
  # Returns true if the databases directory contains one or more incompatible
71
60
  # databases.
72
61
  #
73
- # Note that it is okay to only use V4 databases or only V5 databases. It is
62
+ # Note that it is okay to only use V4 databases or only V5 databases.
74
63
  # Incompatibility arises when they are mixed.
75
64
  def any_incompatible?
76
- return false if @fastas_to_reformat.empty?
77
- # We need to compare @fastas_to_reformat to @formatted_fastas, but the
78
- # latter contains extra attributes. However, the first attribute, i.e,
79
- # path is common to both and sufficient for our needs.
80
- to_reformat = @fastas_to_reformat.map(&:first)
81
- formatted = @formatted_fastas.map(&:first)
82
- # Check that they are not equal. Using intersection operator ensures
83
- # comparison even if their order differs.
84
- formatted & to_reformat != formatted
65
+ return false if @formatted_fastas.all? { |ff| ff.v4? || ff.alias? }
66
+ return false if @formatted_fastas.all? { |ff| ff.v5? || ff.alias? }
67
+ true
85
68
  end
86
69
 
87
70
  # Runs makeblastdb on each file in `@fastas_to_format` and
@@ -109,8 +92,8 @@ module SequenceServer
109
92
  # Make the intent clear as well as ensure the program won't crash if
110
93
  # we accidentally call reformat before calling scan.
111
94
  return unless @fastas_to_reformat
112
- @fastas_to_reformat.select do |path, title, type|
113
- make_blast_database('reformat', path, title, type)
95
+ @fastas_to_reformat.select do |path, title, type, non_parse_seqids|
96
+ make_blast_database('reformat', path, title, type, non_parse_seqids)
114
97
  end
115
98
  end
116
99
 
@@ -120,34 +103,36 @@ module SequenceServer
120
103
  # formatted. Adds to @formatted_fastas.
121
104
  def determine_formatted_fastas
122
105
  blastdbcmd.each_line do |line|
123
- path, title, type, *rest = line.split(' ')
106
+ path, *rest = line.chomp.split("\t")
124
107
  next if multipart_database_name?(path)
125
- @formatted_fastas << [path, title, type.strip.downcase, *rest]
108
+ rest << get_categories(path)
109
+ @formatted_fastas << Database.new(path, *rest)
126
110
  end
127
111
  end
128
112
 
129
113
  # Determines which FASTA files in the database directory require
130
114
  # reformatting. Adds to @fastas_to_format.
131
115
  def determine_fastas_to_reformat
132
- @formatted_fastas.each do |path, title, type|
133
- required_extensions = REQUIRED_EXTENSIONS[type]
134
- exts = Dir["#{path}.*"].map { |p| p.split('.').last }.sort
135
- next if (exts & required_extensions) == required_extensions
136
-
137
- @fastas_to_reformat << [path, title, type]
116
+ @formatted_fastas.each do |ff|
117
+ if ff.v4? || ff.non_parse_seqids?
118
+ @fastas_to_reformat << [ff.path, ff.title, ff.type, ff.non_parse_seqids?]
119
+ end
138
120
  end
139
121
  end
140
122
 
141
123
  # Determines which FASTA files in the database directory are
142
124
  # unformatted. Adds to @fastas_to_format.
143
125
  def determine_unformatted_fastas
144
- Find.find(database_dir) do |path|
126
+ # Add a trailing slash to database_dir - Find.find doesn't work as
127
+ # expected without the trailing slash if database_dir is a symlink
128
+ # inside a docker container.
129
+ Find.find(database_dir + '/') do |path|
145
130
  next if File.directory?(path)
146
131
  next unless probably_fasta?(path)
147
132
  next if @formatted_fastas.any? { |f| f[0] == path }
148
133
 
149
134
  @fastas_to_format << [path,
150
- make_db_title(File.basename(path)),
135
+ make_db_title(path),
151
136
  guess_sequence_type_in_fasta(path)]
152
137
  end
153
138
  end
@@ -157,7 +142,7 @@ module SequenceServer
157
142
  # by `determine_formatted_fastas`.
158
143
  def blastdbcmd
159
144
  cmd = "blastdbcmd -recursive -list #{config[:database_dir]}" \
160
- ' -list_outfmt "%f %t %p %n %l %d"'
145
+ ' -list_outfmt "%f %t %p %n %l %d %v"'
161
146
  out, err = sys(cmd, path: config[:bin])
162
147
  errpat = /BLAST Database error/
163
148
  fail BLAST_DATABASE_ERROR.new(cmd, err) if err.match(errpat)
@@ -167,10 +152,11 @@ module SequenceServer
167
152
  end
168
153
 
169
154
  # Create BLAST database, given FASTA file and sequence type in FASTA file.
170
- def make_blast_database(action, file, title, type)
155
+ def make_blast_database(action, file, title, type, non_parse_seqids = false)
171
156
  return unless make_blast_database?(action, file, type)
172
157
  title = confirm_database_title(title)
173
- taxonomy = taxid_map(file) || taxid
158
+ extract_fasta(file) unless File.exist?(file)
159
+ taxonomy = taxid_map(file, non_parse_seqids) || taxid
174
160
  _make_blast_database(file, type, title, taxonomy)
175
161
  end
176
162
 
@@ -200,11 +186,10 @@ module SequenceServer
200
186
 
201
187
  # Check if a '.taxid_map.txt' file exists. If not, try getting it
202
188
  # using blastdbcmd.
203
- def taxid_map(db)
189
+ def taxid_map(db, non_parse_seqids)
190
+ return if non_parse_seqids
204
191
  taxid_map = db.sub(/#{File.extname(db)}$/, '.taxid_map.txt')
205
- if !File.exist?(taxid_map) || File.zero?(taxid_map)
206
- extract_taxid_map(db, taxid_map)
207
- end
192
+ extract_taxid_map(db, taxid_map) if !File.exist?(taxid_map)
208
193
  "-taxid_map #{taxid_map}" if !File.zero?(taxid_map)
209
194
  end
210
195
 
@@ -213,22 +198,23 @@ module SequenceServer
213
198
  # Using 0 as taxid is equivalent to not setting taxid for the database
214
199
  # that will be created.
215
200
  def taxid
201
+ default = 0
216
202
  print 'Enter taxid (optional): '
217
203
  user_response = STDIN.gets.strip
218
- "-taxid #{user_response ? Integer(user_response) : 0}"
204
+ "-taxid #{user_response.empty? && default || Integer(user_response)}"
219
205
  rescue ArgumentError # presumably from call to Interger()
220
206
  puts 'taxid should be a number'
221
207
  retry
222
208
  end
223
209
 
224
210
  def _make_blast_database(file, type, title, taxonomy)
225
- extract_fasta(file) unless File.exist?(file)
226
- cmd = "makeblastdb -parse_seqids -hash_index -in #{file} " \
227
- "-dbtype #{type.to_s.slice(0, 4)} -title '#{title}'" \
211
+ cmd = "makeblastdb -parse_seqids -hash_index -in '#{file}'" \
212
+ " -dbtype #{type.to_s.slice(0, 4)} -title '#{title}'" \
228
213
  " #{taxonomy}"
229
214
  out, err = sys(cmd, path: config[:bin])
230
215
  puts out.strip
231
216
  puts err.strip
217
+ return true
232
218
  rescue CommandFailed => e
233
219
  puts <<~MSG
234
220
  Could not create BLAST database for: #{file}
@@ -278,8 +264,16 @@ module SequenceServer
278
264
  !(db_name.match(%r{.+/\S+\.\d{2,3}$}).nil?)
279
265
  end
280
266
 
267
+ def get_categories(path)
268
+ path
269
+ .gsub(config[:database_dir], '') # remove database_dir from path
270
+ .split('/')
271
+ .reject(&:empty?)[0..-2] # the first entry might be '' if database_dir does not end with /
272
+ end
273
+
281
274
  # Returns true if first character of the file is '>'.
282
275
  def probably_fasta?(file)
276
+ return false unless file.match(/((cds)|(fasta)|(fna)|(pep)|(cdna)|(fa)|(prot)|(fas)|(genome)|(nuc)|(dna)|(nt))$/i)
283
277
  File.read(file, 1) == '>'
284
278
  end
285
279
 
@@ -288,7 +282,8 @@ module SequenceServer
288
282
  # For example:
289
283
  # Cobs1.4.proteins.fasta -> Cobs 1.4 proteins
290
284
  # S_invicta.xx.2.5.small.nucl.fa -> S invicta xx 2.5 small nucl
291
- def make_db_title(db_name)
285
+ def make_db_title(path)
286
+ db_name = File.basename(path)
292
287
  db_name.tr!('"', "'")
293
288
  # removes .fasta like extension names
294
289
  db_name.gsub!(File.extname(db_name), '')
@@ -79,6 +79,10 @@ module SequenceServer
79
79
  options: SequenceServer.config[:options]
80
80
  }
81
81
 
82
+ if SequenceServer.config[:databases_widget] == 'tree'
83
+ searchdata.update(tree: Database.tree)
84
+ end
85
+
82
86
  # If a job_id is specified, update searchdata from job meta data (i.e.,
83
87
  # query, pre-selected databases, advanced options used). Query is only
84
88
  # updated if params[:query] is not specified.
@@ -1,4 +1,4 @@
1
1
  # Define version number.
2
2
  module SequenceServer
3
- VERSION = '2.0.0.rc8'.freeze
3
+ VERSION = '2.0.0'.freeze
4
4
  end
@@ -5,11 +5,7 @@ require 'resolv'
5
5
  # Top level module / namespace.
6
6
  module SequenceServer
7
7
  # The default version of BLAST that will be downloaded and configured for use.
8
- BLAST_VERSION = '2.10.0+'.freeze
9
- # The minimum version of BLAST that SequenceServer is happy to run with. This
10
- # is for compatiblity with older database formats. Users will download BLAST
11
- # themselves.
12
- MIN_BLAST_VERSION = '2.9.0+'.freeze
8
+ BLAST_VERSION = '2.12.0+'.freeze
13
9
 
14
10
  # Default location of configuration file.
15
11
  DEFAULT_CONFIG_FILE = '~/.sequenceserver.conf'.freeze
@@ -78,9 +74,9 @@ module SequenceServer
78
74
  Thread.abort_on_exception = true if development?
79
75
 
80
76
  # Now locate binaries, scan databases directory, require any plugin files.
77
+ load_extension
81
78
  init_binaries
82
79
  init_database
83
- load_extension
84
80
 
85
81
  # The above methods validate bin dir, database dir, and path to plugin
86
82
  # files. Port and host settings don't need to be validated: if running
@@ -141,7 +137,8 @@ module SequenceServer
141
137
  puts " - http://#{ip_address}:#{config[:port]}"
142
138
  puts " - http://#{hostname}:#{config[:port]}" if hostname
143
139
  puts ' To share your setup with anyone in the world, ask your IT team'
144
- puts ' for a public IP address.'
140
+ puts ' for a public IP address or consider the SequenceServer cloud'
141
+ puts ' hosting service: https://sequenceserver.com/cloud'
145
142
  puts ' To disable sharing, set :host: key in config file to 127.0.0.1'
146
143
  puts ' and restart server.'
147
144
  end
@@ -205,12 +202,19 @@ module SequenceServer
205
202
 
206
203
  makeblastdb.scan
207
204
  fail NO_BLAST_DATABASE_FOUND, config[:database_dir] if !makeblastdb.any_formatted?
208
- fail INCOMPATIBLE_BLAST_DATABASES, config[:database_dir] if makeblastdb.any_incompatible?
209
205
 
210
206
  Database.collection = makeblastdb.formatted_fastas
211
207
  Database.each do |database|
212
- logger.debug("Found #{database.type} database '#{database.title}'" \
213
- " at '#{database.name}'")
208
+ logger.debug "Found #{database.type} database '#{database.title}' at '#{database.path}'"
209
+ if database.non_parse_seqids?
210
+ logger.warn "Database '#{database.title}' was created without using the" \
211
+ ' -parse_seqids option of makeblastdb. FASTA download will' \
212
+ " not work correctly (path: '#{database.path}')."
213
+ elsif database.v4?
214
+ logger.warn "Database '#{database.title}' is of older format. Mixing" \
215
+ ' old and new format databases can be problematic' \
216
+ "(path: '#{database.path}')."
217
+ end
214
218
  end
215
219
  end
216
220
 
@@ -245,7 +249,7 @@ module SequenceServer
245
249
  end
246
250
  version = out.split[1]
247
251
  fail BLAST_NOT_INSTALLED_OR_NOT_EXECUTABLE if version.empty?
248
- fail BLAST_NOT_COMPATIBLE, version unless is_compatible(version, MIN_BLAST_VERSION)
252
+ fail BLAST_NOT_COMPATIBLE, version unless is_compatible(version, BLAST_VERSION)
249
253
  end
250
254
 
251
255
  def server_url