sequenceserver 2.0.0.rc8 → 2.0.0

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of sequenceserver might be problematic. Click here for more details.

Files changed (116) hide show
  1. checksums.yaml +4 -4
  2. data/bin/sequenceserver +22 -30
  3. data/lib/sequenceserver/api_errors.rb +5 -1
  4. data/lib/sequenceserver/blast/constants.rb +1 -1
  5. data/lib/sequenceserver/blast/hit.rb +5 -16
  6. data/lib/sequenceserver/blast/job.rb +9 -18
  7. data/lib/sequenceserver/blast/report.rb +5 -3
  8. data/lib/sequenceserver/config.rb +4 -1
  9. data/lib/sequenceserver/database.rb +69 -9
  10. data/lib/sequenceserver/job.rb +1 -1
  11. data/lib/sequenceserver/makeblastdb.rb +40 -45
  12. data/lib/sequenceserver/routes.rb +4 -0
  13. data/lib/sequenceserver/version.rb +1 -1
  14. data/lib/sequenceserver.rb +15 -11
  15. data/public/config.js +143 -142
  16. data/public/css/fonts.css +23 -22
  17. data/public/css/grapher.css +598 -594
  18. data/public/css/sequenceserver.css +86 -24
  19. data/public/css/sequenceserver.min.css +2 -2
  20. data/public/js/alignment_exporter.js +14 -14
  21. data/public/js/databases_tree.js +215 -0
  22. data/public/js/download_fasta.js +1 -1
  23. data/public/js/hit.js +6 -2
  24. data/public/js/hits_overview.js +1 -1
  25. data/public/js/length_distribution.js +5 -5
  26. data/public/js/query.js +4 -7
  27. data/public/js/report.js +12 -24
  28. data/public/js/search.js +21 -2
  29. data/public/js/sidebar.js +4 -4
  30. data/public/js/svgExporter.js +12 -12
  31. data/public/js/visualisation_helpers.js +4 -5
  32. data/public/sequenceserver-report.min.js +11 -11
  33. data/public/sequenceserver-search.min.js +15 -11
  34. data/public/vendor/github/vakata/jstree@3.3.8/LICENSE-MIT +22 -0
  35. data/public/vendor/github/vakata/jstree@3.3.8/README.md +663 -0
  36. data/public/vendor/github/vakata/jstree@3.3.8/bower.json +33 -0
  37. data/public/vendor/github/vakata/jstree@3.3.8/component.json +28 -0
  38. data/public/vendor/github/vakata/jstree@3.3.8/composer.json +46 -0
  39. data/public/vendor/github/vakata/jstree@3.3.8/demo/README.md +2 -0
  40. data/public/vendor/github/vakata/jstree@3.3.8/demo/basic/index.html +146 -0
  41. data/public/vendor/github/vakata/jstree@3.3.8/demo/basic/root.json +1 -0
  42. data/public/vendor/github/vakata/jstree@3.3.8/dist/jstree.js +8612 -0
  43. data/public/vendor/github/vakata/jstree@3.3.8/dist/jstree.min.js +6 -0
  44. data/public/vendor/github/vakata/jstree@3.3.8/dist/themes/default/32px.png +0 -0
  45. data/public/vendor/github/vakata/jstree@3.3.8/dist/themes/default/40px.png +0 -0
  46. data/public/vendor/github/vakata/jstree@3.3.8/dist/themes/default/style.css +1102 -0
  47. data/public/vendor/github/vakata/jstree@3.3.8/dist/themes/default/style.min.css +1 -0
  48. data/public/vendor/github/vakata/jstree@3.3.8/dist/themes/default/throbber.gif +0 -0
  49. data/public/vendor/github/vakata/jstree@3.3.8/dist/themes/default-dark/32px.png +0 -0
  50. data/public/vendor/github/vakata/jstree@3.3.8/dist/themes/default-dark/40px.png +0 -0
  51. data/public/vendor/github/vakata/jstree@3.3.8/dist/themes/default-dark/style.css +1146 -0
  52. data/public/vendor/github/vakata/jstree@3.3.8/dist/themes/default-dark/style.min.css +1 -0
  53. data/public/vendor/github/vakata/jstree@3.3.8/dist/themes/default-dark/throbber.gif +0 -0
  54. data/public/vendor/github/vakata/jstree@3.3.8/gruntfile.js +242 -0
  55. data/public/vendor/github/vakata/jstree@3.3.8/jstree.jquery.json +28 -0
  56. data/public/vendor/github/vakata/jstree@3.3.8/package.json +58 -0
  57. data/public/vendor/github/vakata/jstree@3.3.8/src/intro.js +14 -0
  58. data/public/vendor/github/vakata/jstree@3.3.8/src/jstree.changed.js +69 -0
  59. data/public/vendor/github/vakata/jstree@3.3.8/src/jstree.checkbox.js +976 -0
  60. data/public/vendor/github/vakata/jstree@3.3.8/src/jstree.conditionalselect.js +38 -0
  61. data/public/vendor/github/vakata/jstree@3.3.8/src/jstree.contextmenu.js +661 -0
  62. data/public/vendor/github/vakata/jstree@3.3.8/src/jstree.dnd.js +669 -0
  63. data/public/vendor/github/vakata/jstree@3.3.8/src/jstree.js +4931 -0
  64. data/public/vendor/github/vakata/jstree@3.3.8/src/jstree.massload.js +137 -0
  65. data/public/vendor/github/vakata/jstree@3.3.8/src/jstree.search.js +421 -0
  66. data/public/vendor/github/vakata/jstree@3.3.8/src/jstree.sort.js +74 -0
  67. data/public/vendor/github/vakata/jstree@3.3.8/src/jstree.state.js +138 -0
  68. data/public/vendor/github/vakata/jstree@3.3.8/src/jstree.types.js +372 -0
  69. data/public/vendor/github/vakata/jstree@3.3.8/src/jstree.unique.js +164 -0
  70. data/public/vendor/github/vakata/jstree@3.3.8/src/jstree.wholerow.js +122 -0
  71. data/public/vendor/github/vakata/jstree@3.3.8/src/misc.js +656 -0
  72. data/public/vendor/github/vakata/jstree@3.3.8/src/outro.js +1 -0
  73. data/public/vendor/github/vakata/jstree@3.3.8/src/sample.js +93 -0
  74. data/public/vendor/github/vakata/jstree@3.3.8/src/themes/base.less +93 -0
  75. data/public/vendor/github/vakata/jstree@3.3.8/src/themes/default/32px.png +0 -0
  76. data/public/vendor/github/vakata/jstree@3.3.8/src/themes/default/40px.png +0 -0
  77. data/public/vendor/github/vakata/jstree@3.3.8/src/themes/default/style.css +1102 -0
  78. data/public/vendor/github/vakata/jstree@3.3.8/src/themes/default/style.less +22 -0
  79. data/public/vendor/github/vakata/jstree@3.3.8/src/themes/default/throbber.gif +0 -0
  80. data/public/vendor/github/vakata/jstree@3.3.8/src/themes/default-dark/32px.png +0 -0
  81. data/public/vendor/github/vakata/jstree@3.3.8/src/themes/default-dark/40px.png +0 -0
  82. data/public/vendor/github/vakata/jstree@3.3.8/src/themes/default-dark/style.css +1146 -0
  83. data/public/vendor/github/vakata/jstree@3.3.8/src/themes/default-dark/style.less +50 -0
  84. data/public/vendor/github/vakata/jstree@3.3.8/src/themes/default-dark/throbber.gif +0 -0
  85. data/public/vendor/github/vakata/jstree@3.3.8/src/themes/main.less +77 -0
  86. data/public/vendor/github/vakata/jstree@3.3.8/src/themes/mixins.less +104 -0
  87. data/public/vendor/github/vakata/jstree@3.3.8/src/themes/responsive.less +67 -0
  88. data/public/vendor/github/vakata/jstree@3.3.8/src/vakata-jstree.js +38 -0
  89. data/public/vendor/github/vakata/jstree@3.3.8/test/unit/index.html +16 -0
  90. data/public/vendor/github/vakata/jstree@3.3.8/test/unit/libs/qunit.css +244 -0
  91. data/public/vendor/github/vakata/jstree@3.3.8/test/unit/libs/qunit.js +2212 -0
  92. data/public/vendor/github/vakata/jstree@3.3.8/test/unit/test.js +11 -0
  93. data/public/vendor/github/vakata/jstree@3.3.8/test/visual/desktop/index.html +44 -0
  94. data/public/vendor/github/vakata/jstree@3.3.8/test/visual/mobile/index.html +42 -0
  95. data/public/vendor/github/vakata/jstree@3.3.8/test/visual/screenshots/desktop/desktop.png +0 -0
  96. data/public/vendor/github/vakata/jstree@3.3.8/test/visual/screenshots/desktop/home.png +0 -0
  97. data/public/vendor/github/vakata/jstree@3.3.8/test/visual/screenshots/mobile/home.png +0 -0
  98. data/public/vendor/github/vakata/jstree@3.3.8/test/visual/screenshots/mobile/mobile.png +0 -0
  99. data/public/vendor/github/vakata/jstree@3.3.8.js +3 -0
  100. data/public/vendor/system-csp-production.js +3 -3
  101. data/public/vendor/system-csp-production.js.map +1 -1
  102. data/public/vendor/system-csp-production.src.js +146 -140
  103. data/public/vendor/system-polyfills.js.map +1 -1
  104. data/public/vendor/system-polyfills.src.js +1 -0
  105. data/public/vendor/system.js +3 -3
  106. data/public/vendor/system.js.map +1 -1
  107. data/public/vendor/system.src.js +4771 -2383
  108. data/views/_options.erb +21 -0
  109. data/views/layout.erb +17 -18
  110. metadata +102 -43
  111. data/bin/chromedriver +0 -0
  112. data/bin/geckodriver +0 -0
  113. data/public/shims/form-core.js +0 -3
  114. data/public/shims/form-validation.js +0 -3
  115. data/public/shims/plugins/jquery.ui.position.js +0 -13
  116. data/public/shims/styles/shim.css +0 -1
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 449efaa2619fd2c83a4116480c78ae6b892804f2f9bc23c3d074b8b0493985c5
4
- data.tar.gz: 393f9da8bd3df53bff3ed3a137a5d9484ad5ec9cfa1de3c7b15c3f1a6833f092
3
+ metadata.gz: 25c2424857780517682276d21694f8ab0c3594772b82f08dcdca1934dcf23fd9
4
+ data.tar.gz: 3e27329d829825b7a3a73f14ebdecad514648381dbba914cfca6ee63564c795a
5
5
  SHA512:
6
- metadata.gz: de2fdcef61a8fcdbe8907935a1f0b545219698fe9d37fcad6cd40aaa50e779010b7e91a9158eb76590baed78ce3743cdc173b664ff5d0969dfaca42eb6bd621c
7
- data.tar.gz: a65c243d19435021b00a360ddfa8f617fa58e5944dd55117a8865f11dec4554a86e577cdfee3013a82638dc45de697a33156da1ba0a2fedd8e2936a7d760d5d9
6
+ metadata.gz: 5137bf6118ee0044a0b8d5b1979f525686d7dd1698d6c795682d39a6c9d8247410104441fb642a585e6df9a18ad827336c8f61215f9e3c8c68fb387e0b6e2f41
7
+ data.tar.gz: e7ec5cebd2595b3a5f90cb750ce583161821e1be22dfcf14b1ba1fbe11dda440105795ac04b06caeb456c0f78e55db50d730704db8972667ca8d7acae5170692
data/bin/sequenceserver CHANGED
@@ -308,34 +308,6 @@ begin
308
308
  exit! unless set?
309
309
  end
310
310
  end
311
- rescue SequenceServer::INCOMPATIBLE_BLAST_DATABASES => e
312
- unless list_databases? || make_blast_databases?
313
- # Print error raised.
314
- puts
315
- puts e
316
-
317
- # Offer user to format the FASTA files.
318
- database_dir = SequenceServer.config[:database_dir]
319
- puts
320
- puts <<~MSG
321
- Incompatible databases can cause BLAST searches and other features of
322
- SequenceServer to fail unexpectedly.
323
- You can view incompatible databases and choose to reformat them below.
324
- Alternatively, please remove them from databases directory.
325
-
326
- View incompatible databases? [y/n] (Default: y).
327
- MSG
328
- puts
329
- print '>> '
330
- response = STDIN.gets.to_s.strip
331
- unless response =~ /^[n]$/i
332
- reformatted = SequenceServer.makeblastdb.reformat
333
- exit! if reformatted.empty? && !set?
334
- redo unless set?
335
- else
336
- exit! unless set?
337
- end
338
- end
339
311
  rescue SequenceServer::ENOENT,
340
312
  SequenceServer::CONFIG_FILE_ERROR,
341
313
  SequenceServer::BLAST_DATABASE_ERROR,
@@ -382,8 +354,28 @@ begin
382
354
 
383
355
  if make_blast_databases?
384
356
  if SequenceServer.makeblastdb.scan
385
- SequenceServer.makeblastdb.run
386
- else
357
+ puts
358
+ puts <<~MSG
359
+ SequenceServer has scanned your databases directory and will now offer
360
+ to convert FASTA files into BLAST databases. It will also offer to
361
+ reformat any old-format BLAST databases and those created without
362
+ the -parse_seqids option of makeblastdb (-parse_seqids option is
363
+ required for sequence retrieval to correctly work).
364
+
365
+ Note that reformatting process can be slow large BLAST databases and
366
+ fail if sequence identifiers are longer than 50 characters. While we
367
+ exepect the reformatting process to work in most other cases, things
368
+ can inevitably go wrong. Thus, please back up your databases before
369
+ reformatting and post any issues to our Google Group/GitHub so that
370
+ we can all learn from it.
371
+
372
+ Proceed? [y/n] (Default: y).
373
+ MSG
374
+ puts
375
+ print '>> '
376
+ response = STDIN.gets.to_s.strip
377
+ SequenceServer.makeblastdb.run unless response =~ /^[n]$/i
378
+ else
387
379
  puts "All FASTA files in #{SequenceServer.config[:database_dir]} " \
388
380
  'are formatted.'
389
381
  end
@@ -41,7 +41,11 @@ module SequenceServer
41
41
  def message
42
42
  <<~MSG
43
43
  Looks like there's a problem with one of the query sequences, selected
44
- databases, or advanced parameters.
44
+ databases, or advanced parameters. Details of the error are included
45
+ below. Please ask on our
46
+ <a href="https://github.com/wurmlab/sequenceserver/issues" target="_blank">issue tracker</a>
47
+ or on our <a href="https://groups.google.com/g/sequenceserver">forum</a> if you are
48
+ not sure what the error message means, or if the error message is just a number.
45
49
  MSG
46
50
  end
47
51
 
@@ -2,7 +2,7 @@
2
2
  module SequenceServer
3
3
  # Define constanst used by BLAST module.
4
4
  module BLAST
5
- ERROR_LINE = /Error:\s(.*)/
5
+ ERROR_LINE = /Error:\s(.*)/i
6
6
 
7
7
  ALGORITHMS = %w[blastn blastp blastx tblastn tblastx].freeze
8
8
 
@@ -21,8 +21,8 @@ module SequenceServer
21
21
  # representation.
22
22
  def to_json(*args)
23
23
  # List all attributes that we want to send to the browser.
24
- properties = %i[number id accession title length score identity
25
- qcovs sciname evalue hsps links]
24
+ properties = %i[number id accession title length total_score
25
+ qcovs sciname hsps links]
26
26
  properties.inject({}) { |h, k| h[k] = send(k); h }.to_json(*args)
27
27
  end
28
28
 
@@ -71,23 +71,12 @@ module SequenceServer
71
71
  # client.
72
72
  ###
73
73
 
74
- # Returns the sum of scores of all HSPs.
75
- def score
74
+ # Returns the sum of scores of all HSPs. Displayed in the tabular summary
75
+ # of hits in the HTML report. Should probably be calculated in browser?
76
+ def total_score
76
77
  hsps.map(&:score).reduce(:+)
77
78
  end
78
79
 
79
- # Returns the sum of identity of all HSPs divided by sum of length of all
80
- # HSPs (expressed as percentage).
81
- def identity
82
- hsps.map(&:identity).reduce(:+) * 100 / hsps.map(&:length).reduce(:+)
83
- end
84
-
85
- # Returns the minimum evalue of all HSPs of the Hit. This is shown in the
86
- # tabular overview of hits in the HTML report.
87
- def evalue
88
- hsps.first.evalue
89
- end
90
-
91
80
  private
92
81
 
93
82
  # Returns the report object that this hit is a part of. This is used to
@@ -60,28 +60,19 @@ module SequenceServer
60
60
 
61
61
  # Handle error. See [1].
62
62
  case exitstatus
63
- when 1
64
- # Error in query sequence or options.
63
+ when 1..2
64
+ # 1: Error in query sequences or options.
65
+ # 2: Error in BLAST databases.
65
66
  error = IO.foreach(stderr).grep(ERROR_LINE).join
66
67
  error = File.read(stderr) if error.empty?
67
- fail InputError, error
68
- when 2
69
- fail InputError, <<~MSG
70
- BLAST signalled a problem with the databases that you searched.
71
-
72
- Most likely one or more of your databases were created using an
73
- older version of BLAST. Please consider recreating the databases
74
- using BLAST #{BLAST_VERSION}.
75
-
76
- As a temporary solution, you can try searching one database at a time.
77
- MSG
68
+ fail InputError, "(#{exitstatus}) #{error}"
78
69
  when 4
79
70
  # Out of memory. User can retry with a shorter search, so raising
80
71
  # InputError here instead of SystemError.
81
72
  fail InputError, <<~MSG
82
- Ran out of memory. Please try a smaller query, or searching fewer and smaller
73
+ Ran out of memory. Please try a smaller query, fewer and smaller
83
74
  databases, or limiting the output by using advanced options.
84
- MSG
75
+ MSG
85
76
  when 6
86
77
  # Error creating output files. It can't be a permission issue as that
87
78
  # would have been caught while creating job directory. But we can run
@@ -98,7 +89,7 @@ module SequenceServer
98
89
  fail SystemError, <<~MSG
99
90
  BLAST failed abruptly (exit status: #{exitstatus}). Most likely there is a
100
91
  problem with the BLAST+ binaries.
101
- MSG
92
+ MSG
102
93
  end
103
94
  end
104
95
  # rubocop:enable Metrics/CyclomaticComplexity
@@ -155,7 +146,7 @@ module SequenceServer
155
146
  end
156
147
 
157
148
  def allowed_chars
158
- /\A[a-z0-9\-_\. ']*\Z/i
149
+ /\A[a-z0-9\-_\. ',]*\Z/i
159
150
  end
160
151
 
161
152
  def disallowed_options
@@ -167,4 +158,4 @@ end
167
158
 
168
159
  # References
169
160
  # ----------
170
- # [1]: http://www.ncbi.nlm.nih.gov/books/NBK1763/
161
+ # [1]: http://www.ncbi.nlm.nih.gov/books/NBK1763/ (Appendices)
@@ -25,6 +25,7 @@ module SequenceServer
25
25
  def initialize(job)
26
26
  super do
27
27
  @queries = []
28
+ @querydb = job.databases
28
29
  end
29
30
  end
30
31
 
@@ -42,7 +43,8 @@ module SequenceServer
42
43
  }.update(search_id: job.id,
43
44
  submitted_at: job.submitted_at.utc,
44
45
  imported_xml: !!job.imported_xml_file,
45
- seqserv_version: SequenceServer::VERSION).to_json
46
+ seqserv_version: SequenceServer::VERSION,
47
+ non_parse_seqids: !!job.databases&.any?(&:non_parse_seqids?)).to_json
46
48
  end
47
49
 
48
50
  private
@@ -80,13 +82,12 @@ module SequenceServer
80
82
  # Get database information (title and type) from job yaml or from XML.
81
83
  # Sets `querydb` and `dbtype` attributes.
82
84
  def extract_db_info(ir)
83
- if job.databases.empty?
85
+ if @querydb.empty?
84
86
  @querydb = ir[3].split.map do |path|
85
87
  { title: File.basename(path) }
86
88
  end
87
89
  @dbtype = dbtype_from_program
88
90
  else
89
- @querydb = job.databases
90
91
  @dbtype = @querydb.first.type
91
92
  end
92
93
  end
@@ -176,6 +177,7 @@ module SequenceServer
176
177
  def parse_xml(xml)
177
178
  node_to_array Ox.parse(xml).root
178
179
  rescue Ox::ParseError
180
+ fail 'Error parsing XML file' if job.imported_xml_file
179
181
  fail InputError, <<~MSG
180
182
  BLAST generated incorrect XML output. This can happen if sequence ids in your
181
183
  databases are not unique across all files. As a temporary workaround, you can
@@ -99,6 +99,7 @@ module SequenceServer
99
99
  {
100
100
  host: '0.0.0.0',
101
101
  port: 4567,
102
+ databases_widget: 'classic',
102
103
  options: {
103
104
  blastn: ['-task blastn', '-evalue 1e-5'],
104
105
  blastp: ['-evalue 1e-5'],
@@ -106,7 +107,9 @@ module SequenceServer
106
107
  tblastx: ['-evalue 1e-5'],
107
108
  tblastn: ['-evalue 1e-5']
108
109
  },
109
- num_threads: 1
110
+ num_threads: 1,
111
+ num_jobs: 1,
112
+ job_lifetime: 43200
110
113
  }
111
114
  end
112
115
  end
@@ -19,14 +19,14 @@ module SequenceServer
19
19
  # SequenceServer will always place BLAST database files alongside input FASTA,
20
20
  # and use `parse_seqids` option of `makeblastdb` to format databases.
21
21
  Database = Struct.new(:name, :title, :type, :nsequences, :ncharacters,
22
- :updated_on) do
22
+ :updated_on, :format, :categories) do
23
23
 
24
24
  extend Forwardable
25
25
 
26
26
  def_delegators SequenceServer, :config, :sys
27
27
 
28
28
  def initialize(*args)
29
- args[2].downcase! # database type
29
+ args[2].downcase! # type
30
30
  args.each(&:freeze)
31
31
  super
32
32
 
@@ -34,6 +34,7 @@ module SequenceServer
34
34
  end
35
35
 
36
36
  attr_reader :id
37
+ alias path name
37
38
 
38
39
  def retrieve(accession, coords = nil)
39
40
  cmd = "blastdbcmd -db #{name} -entry '#{accession}'"
@@ -48,10 +49,36 @@ module SequenceServer
48
49
  nil
49
50
  end
50
51
 
51
- def include?(accession)
52
- cmd = "blastdbcmd -entry '#{accession}' -db #{name}"
53
- out, = sys(cmd, path: config[:bin])
54
- !out.empty?
52
+ # Returns true if the database contains the given sequence id.
53
+ # Returns false otherwise.
54
+ def include?(id)
55
+ cmd = "blastdbcmd -entry '#{id}' -db #{name}"
56
+ sys(cmd, path: config[:bin]) rescue false
57
+ end
58
+
59
+ def v4?
60
+ format == '4'
61
+ end
62
+
63
+ def v5?
64
+ format == '5'
65
+ end
66
+
67
+ # Return true if the database was _not_ created using the -parse_seqids
68
+ # option of makeblastdb.
69
+ def non_parse_seqids?
70
+ return if alias?
71
+ case format
72
+ when '5'
73
+ (%w[nog nos pog pos] & extensions).length != 2
74
+ when '4'
75
+ (%w[nog nsd nsi pod psd psi] & extensions).length != 3
76
+ end
77
+ end
78
+
79
+ # Returns true if the database was created using blastdb_aliastool.
80
+ def alias?
81
+ (%w[nal pal] & extensions).length == 1 && extensions.count == 1
55
82
  end
56
83
 
57
84
  def ==(other)
@@ -65,6 +92,16 @@ module SequenceServer
65
92
  def to_json(*args)
66
93
  to_h.update(id: id).to_json(*args)
67
94
  end
95
+
96
+ private
97
+
98
+ def extensions
99
+ # The glob pattern used here is quite relaxed. This is to capture
100
+ # multipart databases as well. It is possible that non-blast-database
101
+ # extensions may also be picked. However, that shouldn't be a problem
102
+ # as we only check whether certain required extensions are present or not.
103
+ @extensions ||= Dir["#{path}*{n,p}*"].map { |p| p.split('.').last }.sort.uniq
104
+ end
68
105
  end
69
106
 
70
107
  # Model Database's eigenclass as a collection of Database objects.
@@ -80,9 +117,8 @@ module SequenceServer
80
117
  @collection ||= {}
81
118
  end
82
119
 
83
- def collection=(databases_attrs)
84
- databases_attrs.each do |db_attrs|
85
- db = Database.new(*db_attrs)
120
+ def collection=(databases)
121
+ databases.each do |db|
86
122
  collection[db.id] = db
87
123
  end
88
124
  end
@@ -102,6 +138,30 @@ module SequenceServer
102
138
  collection.values
103
139
  end
104
140
 
141
+ def tree
142
+ all.each_with_object({}) do |db, data|
143
+ data[db.type] ||= []
144
+ use_parent = '#'
145
+ db.categories.each_with_index do |entry, index|
146
+ parent = index.zero? ? '#' : db.categories[0..(index - 1)].join('-')
147
+ use_id = db.categories[0..index].join('-')
148
+ element = { id: use_id, parent: parent, text: entry }
149
+ data[db.type] << element unless data[db.type].include?(element)
150
+ use_parent = use_id
151
+ end
152
+
153
+ data[db.type] <<
154
+ {
155
+ id: db.id,
156
+ parent: use_parent,
157
+ text: db.title,
158
+ icon: 'glyphicon glyphicon-file'
159
+ }
160
+
161
+ yield(db, data[db.type].last) if block_given?
162
+ end
163
+ end
164
+
105
165
  def each(&block)
106
166
  all.each(&block)
107
167
  end
@@ -51,7 +51,7 @@ module SequenceServer
51
51
 
52
52
  # Thread pool used for running BLAST searches.
53
53
  def pool
54
- @pool ||= Pool.new(SequenceServer.config[:num_threads])
54
+ @pool ||= Pool.new(SequenceServer.config[:num_jobs] || 1)
55
55
  end
56
56
  end
57
57
 
@@ -11,17 +11,6 @@ module SequenceServer
11
11
  # makeblastdb.scan && makeblastdb.run
12
12
  #
13
13
  class MAKEBLASTDB
14
- # We want V5 databases created using -parse_seqids for proper function of
15
- # SequenceServer. This means each database should be comprised of at least 9
16
- # files with the following extensions. Databases created by us will have two
17
- # additional files with the extensions nhd and nhi, or phd and phi, due to
18
- # the use of -hash_index option. Finally, multipart databases will have one
19
- # additional file with the extension nal or pal.
20
- REQUIRED_EXTENSIONS = {
21
- 'nucleotide' => %w{ndb nhr nin nog nos not nsq ntf nto}.freeze,
22
- 'protein' => %w{pdb phr pin pog pos pot psq ptf pto}.freeze
23
- }
24
-
25
14
  extend Forwardable
26
15
 
27
16
  def_delegators SequenceServer, :config, :sys
@@ -70,18 +59,12 @@ module SequenceServer
70
59
  # Returns true if the databases directory contains one or more incompatible
71
60
  # databases.
72
61
  #
73
- # Note that it is okay to only use V4 databases or only V5 databases. It is
62
+ # Note that it is okay to only use V4 databases or only V5 databases.
74
63
  # Incompatibility arises when they are mixed.
75
64
  def any_incompatible?
76
- return false if @fastas_to_reformat.empty?
77
- # We need to compare @fastas_to_reformat to @formatted_fastas, but the
78
- # latter contains extra attributes. However, the first attribute, i.e,
79
- # path is common to both and sufficient for our needs.
80
- to_reformat = @fastas_to_reformat.map(&:first)
81
- formatted = @formatted_fastas.map(&:first)
82
- # Check that they are not equal. Using intersection operator ensures
83
- # comparison even if their order differs.
84
- formatted & to_reformat != formatted
65
+ return false if @formatted_fastas.all? { |ff| ff.v4? || ff.alias? }
66
+ return false if @formatted_fastas.all? { |ff| ff.v5? || ff.alias? }
67
+ true
85
68
  end
86
69
 
87
70
  # Runs makeblastdb on each file in `@fastas_to_format` and
@@ -109,8 +92,8 @@ module SequenceServer
109
92
  # Make the intent clear as well as ensure the program won't crash if
110
93
  # we accidentally call reformat before calling scan.
111
94
  return unless @fastas_to_reformat
112
- @fastas_to_reformat.select do |path, title, type|
113
- make_blast_database('reformat', path, title, type)
95
+ @fastas_to_reformat.select do |path, title, type, non_parse_seqids|
96
+ make_blast_database('reformat', path, title, type, non_parse_seqids)
114
97
  end
115
98
  end
116
99
 
@@ -120,34 +103,36 @@ module SequenceServer
120
103
  # formatted. Adds to @formatted_fastas.
121
104
  def determine_formatted_fastas
122
105
  blastdbcmd.each_line do |line|
123
- path, title, type, *rest = line.split(' ')
106
+ path, *rest = line.chomp.split("\t")
124
107
  next if multipart_database_name?(path)
125
- @formatted_fastas << [path, title, type.strip.downcase, *rest]
108
+ rest << get_categories(path)
109
+ @formatted_fastas << Database.new(path, *rest)
126
110
  end
127
111
  end
128
112
 
129
113
  # Determines which FASTA files in the database directory require
130
114
  # reformatting. Adds to @fastas_to_format.
131
115
  def determine_fastas_to_reformat
132
- @formatted_fastas.each do |path, title, type|
133
- required_extensions = REQUIRED_EXTENSIONS[type]
134
- exts = Dir["#{path}.*"].map { |p| p.split('.').last }.sort
135
- next if (exts & required_extensions) == required_extensions
136
-
137
- @fastas_to_reformat << [path, title, type]
116
+ @formatted_fastas.each do |ff|
117
+ if ff.v4? || ff.non_parse_seqids?
118
+ @fastas_to_reformat << [ff.path, ff.title, ff.type, ff.non_parse_seqids?]
119
+ end
138
120
  end
139
121
  end
140
122
 
141
123
  # Determines which FASTA files in the database directory are
142
124
  # unformatted. Adds to @fastas_to_format.
143
125
  def determine_unformatted_fastas
144
- Find.find(database_dir) do |path|
126
+ # Add a trailing slash to database_dir - Find.find doesn't work as
127
+ # expected without the trailing slash if database_dir is a symlink
128
+ # inside a docker container.
129
+ Find.find(database_dir + '/') do |path|
145
130
  next if File.directory?(path)
146
131
  next unless probably_fasta?(path)
147
132
  next if @formatted_fastas.any? { |f| f[0] == path }
148
133
 
149
134
  @fastas_to_format << [path,
150
- make_db_title(File.basename(path)),
135
+ make_db_title(path),
151
136
  guess_sequence_type_in_fasta(path)]
152
137
  end
153
138
  end
@@ -157,7 +142,7 @@ module SequenceServer
157
142
  # by `determine_formatted_fastas`.
158
143
  def blastdbcmd
159
144
  cmd = "blastdbcmd -recursive -list #{config[:database_dir]}" \
160
- ' -list_outfmt "%f %t %p %n %l %d"'
145
+ ' -list_outfmt "%f %t %p %n %l %d %v"'
161
146
  out, err = sys(cmd, path: config[:bin])
162
147
  errpat = /BLAST Database error/
163
148
  fail BLAST_DATABASE_ERROR.new(cmd, err) if err.match(errpat)
@@ -167,10 +152,11 @@ module SequenceServer
167
152
  end
168
153
 
169
154
  # Create BLAST database, given FASTA file and sequence type in FASTA file.
170
- def make_blast_database(action, file, title, type)
155
+ def make_blast_database(action, file, title, type, non_parse_seqids = false)
171
156
  return unless make_blast_database?(action, file, type)
172
157
  title = confirm_database_title(title)
173
- taxonomy = taxid_map(file) || taxid
158
+ extract_fasta(file) unless File.exist?(file)
159
+ taxonomy = taxid_map(file, non_parse_seqids) || taxid
174
160
  _make_blast_database(file, type, title, taxonomy)
175
161
  end
176
162
 
@@ -200,11 +186,10 @@ module SequenceServer
200
186
 
201
187
  # Check if a '.taxid_map.txt' file exists. If not, try getting it
202
188
  # using blastdbcmd.
203
- def taxid_map(db)
189
+ def taxid_map(db, non_parse_seqids)
190
+ return if non_parse_seqids
204
191
  taxid_map = db.sub(/#{File.extname(db)}$/, '.taxid_map.txt')
205
- if !File.exist?(taxid_map) || File.zero?(taxid_map)
206
- extract_taxid_map(db, taxid_map)
207
- end
192
+ extract_taxid_map(db, taxid_map) if !File.exist?(taxid_map)
208
193
  "-taxid_map #{taxid_map}" if !File.zero?(taxid_map)
209
194
  end
210
195
 
@@ -213,22 +198,23 @@ module SequenceServer
213
198
  # Using 0 as taxid is equivalent to not setting taxid for the database
214
199
  # that will be created.
215
200
  def taxid
201
+ default = 0
216
202
  print 'Enter taxid (optional): '
217
203
  user_response = STDIN.gets.strip
218
- "-taxid #{user_response ? Integer(user_response) : 0}"
204
+ "-taxid #{user_response.empty? && default || Integer(user_response)}"
219
205
  rescue ArgumentError # presumably from call to Interger()
220
206
  puts 'taxid should be a number'
221
207
  retry
222
208
  end
223
209
 
224
210
  def _make_blast_database(file, type, title, taxonomy)
225
- extract_fasta(file) unless File.exist?(file)
226
- cmd = "makeblastdb -parse_seqids -hash_index -in #{file} " \
227
- "-dbtype #{type.to_s.slice(0, 4)} -title '#{title}'" \
211
+ cmd = "makeblastdb -parse_seqids -hash_index -in '#{file}'" \
212
+ " -dbtype #{type.to_s.slice(0, 4)} -title '#{title}'" \
228
213
  " #{taxonomy}"
229
214
  out, err = sys(cmd, path: config[:bin])
230
215
  puts out.strip
231
216
  puts err.strip
217
+ return true
232
218
  rescue CommandFailed => e
233
219
  puts <<~MSG
234
220
  Could not create BLAST database for: #{file}
@@ -278,8 +264,16 @@ module SequenceServer
278
264
  !(db_name.match(%r{.+/\S+\.\d{2,3}$}).nil?)
279
265
  end
280
266
 
267
+ def get_categories(path)
268
+ path
269
+ .gsub(config[:database_dir], '') # remove database_dir from path
270
+ .split('/')
271
+ .reject(&:empty?)[0..-2] # the first entry might be '' if database_dir does not end with /
272
+ end
273
+
281
274
  # Returns true if first character of the file is '>'.
282
275
  def probably_fasta?(file)
276
+ return false unless file.match(/((cds)|(fasta)|(fna)|(pep)|(cdna)|(fa)|(prot)|(fas)|(genome)|(nuc)|(dna)|(nt))$/i)
283
277
  File.read(file, 1) == '>'
284
278
  end
285
279
 
@@ -288,7 +282,8 @@ module SequenceServer
288
282
  # For example:
289
283
  # Cobs1.4.proteins.fasta -> Cobs 1.4 proteins
290
284
  # S_invicta.xx.2.5.small.nucl.fa -> S invicta xx 2.5 small nucl
291
- def make_db_title(db_name)
285
+ def make_db_title(path)
286
+ db_name = File.basename(path)
292
287
  db_name.tr!('"', "'")
293
288
  # removes .fasta like extension names
294
289
  db_name.gsub!(File.extname(db_name), '')
@@ -79,6 +79,10 @@ module SequenceServer
79
79
  options: SequenceServer.config[:options]
80
80
  }
81
81
 
82
+ if SequenceServer.config[:databases_widget] == 'tree'
83
+ searchdata.update(tree: Database.tree)
84
+ end
85
+
82
86
  # If a job_id is specified, update searchdata from job meta data (i.e.,
83
87
  # query, pre-selected databases, advanced options used). Query is only
84
88
  # updated if params[:query] is not specified.
@@ -1,4 +1,4 @@
1
1
  # Define version number.
2
2
  module SequenceServer
3
- VERSION = '2.0.0.rc8'.freeze
3
+ VERSION = '2.0.0'.freeze
4
4
  end
@@ -5,11 +5,7 @@ require 'resolv'
5
5
  # Top level module / namespace.
6
6
  module SequenceServer
7
7
  # The default version of BLAST that will be downloaded and configured for use.
8
- BLAST_VERSION = '2.10.0+'.freeze
9
- # The minimum version of BLAST that SequenceServer is happy to run with. This
10
- # is for compatiblity with older database formats. Users will download BLAST
11
- # themselves.
12
- MIN_BLAST_VERSION = '2.9.0+'.freeze
8
+ BLAST_VERSION = '2.12.0+'.freeze
13
9
 
14
10
  # Default location of configuration file.
15
11
  DEFAULT_CONFIG_FILE = '~/.sequenceserver.conf'.freeze
@@ -78,9 +74,9 @@ module SequenceServer
78
74
  Thread.abort_on_exception = true if development?
79
75
 
80
76
  # Now locate binaries, scan databases directory, require any plugin files.
77
+ load_extension
81
78
  init_binaries
82
79
  init_database
83
- load_extension
84
80
 
85
81
  # The above methods validate bin dir, database dir, and path to plugin
86
82
  # files. Port and host settings don't need to be validated: if running
@@ -141,7 +137,8 @@ module SequenceServer
141
137
  puts " - http://#{ip_address}:#{config[:port]}"
142
138
  puts " - http://#{hostname}:#{config[:port]}" if hostname
143
139
  puts ' To share your setup with anyone in the world, ask your IT team'
144
- puts ' for a public IP address.'
140
+ puts ' for a public IP address or consider the SequenceServer cloud'
141
+ puts ' hosting service: https://sequenceserver.com/cloud'
145
142
  puts ' To disable sharing, set :host: key in config file to 127.0.0.1'
146
143
  puts ' and restart server.'
147
144
  end
@@ -205,12 +202,19 @@ module SequenceServer
205
202
 
206
203
  makeblastdb.scan
207
204
  fail NO_BLAST_DATABASE_FOUND, config[:database_dir] if !makeblastdb.any_formatted?
208
- fail INCOMPATIBLE_BLAST_DATABASES, config[:database_dir] if makeblastdb.any_incompatible?
209
205
 
210
206
  Database.collection = makeblastdb.formatted_fastas
211
207
  Database.each do |database|
212
- logger.debug("Found #{database.type} database '#{database.title}'" \
213
- " at '#{database.name}'")
208
+ logger.debug "Found #{database.type} database '#{database.title}' at '#{database.path}'"
209
+ if database.non_parse_seqids?
210
+ logger.warn "Database '#{database.title}' was created without using the" \
211
+ ' -parse_seqids option of makeblastdb. FASTA download will' \
212
+ " not work correctly (path: '#{database.path}')."
213
+ elsif database.v4?
214
+ logger.warn "Database '#{database.title}' is of older format. Mixing" \
215
+ ' old and new format databases can be problematic' \
216
+ "(path: '#{database.path}')."
217
+ end
214
218
  end
215
219
  end
216
220
 
@@ -245,7 +249,7 @@ module SequenceServer
245
249
  end
246
250
  version = out.split[1]
247
251
  fail BLAST_NOT_INSTALLED_OR_NOT_EXECUTABLE if version.empty?
248
- fail BLAST_NOT_COMPATIBLE, version unless is_compatible(version, MIN_BLAST_VERSION)
252
+ fail BLAST_NOT_COMPATIBLE, version unless is_compatible(version, BLAST_VERSION)
249
253
  end
250
254
 
251
255
  def server_url