labimotion 2.3.0 → 2.4.0.rc11

data/lib/labimotion/libs/converter.rb

@@ -125,6 +125,7 @@ module Labimotion
         tmp_file.rewind
 
         filename = oat.filename
+        # name = "#{File.basename(filename, '.*')}.zip"
         name = "#{File.basename(filename, '.*')}#{File.extname(filename) == '.zip' ? '.bagit.zip' : '.zip'}"
         att = Attachment.new(
           filename: name,
@@ -246,22 +247,14 @@ module Labimotion
       dsr.each do |ds|
         layer = layers[ds[:layer]]
         next if layer.blank? || layer[Labimotion::Prop::FIELDS].blank?
-        is_unit = ds[:field].start_with?(Prop::CONVERTER_FIELD_UINT_PREFIX)
-        next if is_unit && ds[:value].nil?
 
-        field_name = ds[:field].delete_prefix(Prop::CONVERTER_FIELD_UINT_PREFIX)
-
-        fields = layer[Labimotion::Prop::FIELDS].select{ |f| f['field'] == field_name }
+        fields = layer[Labimotion::Prop::FIELDS].select{ |f| f['field'] == ds[:field] }
         fi = fields&.first
         next if fi.blank?
 
         idx = layer[Labimotion::Prop::FIELDS].find_index(fi)
-        if is_unit
-          fi['value_system'] = ds[:value]
-        else
-          fi['value'] = ds[:value]
-          fi['device'] = ds[:device] || ds[:value]
-        end
+        fi['value'] = ds[:value]
+        fi['device'] = ds[:device] || ds[:value]
         new_prop[Labimotion::Prop::LAYERS][ds[:layer]][Labimotion::Prop::FIELDS][idx] = fi
       end
       element = Container.find(dataset.element_id)&.root_element
@@ -327,37 +320,6 @@ module Labimotion
       res
     end
 
-    def self.test_conversions(tmpfile, format)
-      res = {}
-      File.open(tmpfile.path, 'r') do |file|
-        body = { file: file, format: format }
-        response = HTTParty.post(
-          uri('conversions'),
-          basic_auth: auth,
-          body: body,
-          timeout: timeout,
-        )
-        res = response.parsed_response
-      end
-      res
-    end
-
-    def self.restore(profile_id, version, hard)
-      body = { hard: hard }
-      response = HTTParty.post(
-        uri("profiles/restore/#{profile_id}/#{version}"),
-        headers: {
-          "Content-Type" => "application/json"
-        },
-        basic_auth: auth,
-        body: body.to_json,
-        timeout: timeout,
-      )
-      res = response.parsed_response
-
-      res
-    end
-
     def self.metadata(id, current_user)
       att = Attachment.find(id)
       return if att.nil? || att.attachable_id.nil? || att.attachable_type != Labimotion::Prop::CONTAINER
@@ -372,4 +334,4 @@ end
 # rubocop: enable Metrics/AbcSize
 # rubocop: enable Metrics/MethodLength
 # rubocop: enable Metrics/ClassLength
-# rubocop: enable Metrics/CyclomaticComplexity
+# rubocop: enable Metrics/CyclomaticComplexity

data/lib/labimotion/libs/data/datacite/labimotion_template.html.erb

@@ -0,0 +1,67 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<resource xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://datacite.org/schema/kernel-4" xsi:schemaLocation="http://datacite.org/schema/kernel-4 https://schema.datacite.org/meta/kernel-4/metadata.xsd">
+  <identifier identifierType="DOI">https://dx.doi.org/<%= full_doi %></identifier>
+  <creators>
+    <% if creators.empty? %>
+    <creator>
+      <creatorName>(:unav)</creatorName>
+    </creator>
+    <% end %>
+    <% creators.each do |creator| %>
+    <creator>
+      <creatorName nameType="Personal"><%= creator['familyName'] %>, <%= creator['givenName'] %></creatorName>
+      <givenName><%= creator['givenName'] %></givenName>
+      <familyName><%= creator['familyName'] %></familyName>
+      <% if creator['orcid'].present? %>
+      <nameIdentifier schemeURI="https://orcid.org/" nameIdentifierScheme="ORCID"><%= creator['orcid'] %></nameIdentifier>
+      <% end %>
+      <% if creator['affiliation'].present? || creator['ror'].present? %>
+      <affiliation<% if creator['ror'].present? %> affiliationIdentifier="<%= creator['ror'] %>" affiliationIdentifierScheme="ROR" schemeURI="https://ror.org"<% end %>><%= creator['affiliation'] %></affiliation>
+      <% end %>
+    </creator>
+    <% end %>
+  </creators>
+  <titles>
+    <title xml:lang="en-US"><%= title %></title>
+  </titles>
+  <%= render_publisher(publisher_config) %>
+  <publicationYear><%= publication_year %></publicationYear>
+  <subjects>
+    <subject>Chemistry</subject>
+  </subjects>
+  <% if contributors.any? %>
+  <contributors>
+    <% contributors.each do |contributor| %>
+    <contributor contributorType="<%= contributor['contributorType'] %>">
+      <contributorName nameType="Personal"><%= contributor['familyName'] %>, <%= contributor['givenName'] %></contributorName>
+      <givenName><%= contributor['givenName'] %></givenName>
+      <familyName><%= contributor['familyName'] %></familyName>
+      <% if contributor['orcid'].present? %>
+      <nameIdentifier schemeURI="https://orcid.org/" nameIdentifierScheme="ORCID"><%= contributor['orcid'] %></nameIdentifier>
+      <% end %>
+      <% if contributor['affiliation'].present? || contributor['ror'].present? %>
+      <affiliation<% if contributor['ror'].present? %> affiliationIdentifier="<%= contributor['ror'] %>" affiliationIdentifierScheme="ROR" schemeURI="https://ror.org"<% end %>><%= contributor['affiliation'] %></affiliation>
+      <% end %>
+    </contributor>
+    <% end %>
+  </contributors>
+  <% end %>
+  <% if references.any? %>
+  <relatedIdentifiers>
+    <% references.each do |reference| %>
+    <relatedIdentifier relatedIdentifierType="DOI" relationType="References">https://dx.doi.org/<%= reference['doi'] %></relatedIdentifier>
+    <% end %>
+  </relatedIdentifiers>
+  <% end %>
+  <language>en</language>
+  <resourceType resourceTypeGeneral="Model"><%= resource_label %></resourceType>
+  <version><%= version %></version>
+  <rightsList>
+    <rights><%= license %></rights>
+  </rightsList>
+  <descriptions>
+    <% if description.present? %>
+    <description xml:lang="en-US" descriptionType="Abstract"><%= description %></description>
+    <% end %>
+  </descriptions>
+</resource>

data/lib/labimotion/libs/export_element.rb

@@ -1,9 +1,11 @@
 # frozen_string_literal: true
 require 'date'
 require 'ostruct'
+require 'tempfile'
 require 'export_table'
 require 'labimotion/version'
 require 'labimotion/utils/units'
+require 'labimotion/helpers/cover_image_helpers'
 require 'sablon'
 
 module Labimotion
@@ -33,6 +35,7 @@ module Labimotion
       @properties = element.properties
       @options = element.properties_release[Labimotion::Prop::SEL_OPTIONS]
       @export_format = export_format
+      @cover_tempfiles = []
     rescue StandardError => e
       Labimotion.log_exception(e)
     end
@@ -396,6 +399,7 @@ module Labimotion
     TABLE_CELL_TEXT_RENDERERS = {
       Labimotion::FieldType::DRAG_SAMPLE => ->(_sf, c, ctx) { ctx.table_cell_sample_text(c['value'] || {}) },
       Labimotion::FieldType::DRAG_MOLECULE => ->(_sf, c, ctx) { ctx.table_cell_molecule_text(c['value'] || {}) },
+      Labimotion::FieldType::DRAG_ELEMENT => ->(_sf, c, ctx) { ctx.table_cell_element_text(c['value'] || {}) },
       Labimotion::FieldType::SELECT => ->(_sf, c, _ctx) { c['value'].to_s },
       Labimotion::FieldType::SYSTEM_DEFINED => ->(sf, c, ctx) { ctx.table_cell_sysdef_text(sf, c) }
     }.freeze
@@ -430,6 +434,13 @@ module Labimotion
       parts.join("\n")
     end
 
+    def table_cell_element_text(val)
+      parts = []
+      parts << "Short Label: [#{val['el_label']}]" if val['el_label'].present?
+      parts << "Name: [#{val['el_name']}]" if val['el_name'].present?
+      parts.join("\n")
+    end
+
     def table_cell_sysdef_text(sub_field, cell)
       fdef = Labimotion::Units::FIELDS.find { |o| o[:field] == sub_field['option_layers'] }
       unit = fdef&.fetch(:units, [])&.find { |u| u[:key] == cell['value_system'] }&.fetch(:label, '')
@@ -454,10 +465,21 @@ module Labimotion
       # location = Rails.root.join('lib', 'template', 'Labimotion.docx')
       location = Rails.root.join('lib', 'template', 'Labimotion_lines.docx')
       # location = Rails.root.join('lib', 'template', 'Labimotion_img.docx')
-      File.exist?(location)
       template = Sablon.template(location)
-      layers = build_layers
-      context = {
+      tempfile = Tempfile.new('labimotion.docx')
+      template.render_to_file File.expand_path(tempfile), docx_context
+      File.read(tempfile)
+    rescue StandardError => e
+      Labimotion.log_exception(e)
+    ensure
+      # Close and delete the temporary files
+      tempfile&.close
+      tempfile&.unlink
+      close_cover_tempfiles
+    end
+
+    def docx_context
+      {
         label: @element_klass.label,
         desc: @element_klass.desc,
         name: @name,
@@ -465,18 +487,111 @@ module Labimotion
         short_label: @short_label,
         date: Time.now.strftime('%d/%m/%Y'),
         author: @current_user.name,
-        layers: layers
-      }
-      tempfile = Tempfile.new('labimotion.docx')
-      template.render_to_file File.expand_path(tempfile), context
-      content = File.read(tempfile)
-      content
+        layers: build_layers
+      }.merge(cover_context)
+    end
+
+    def close_cover_tempfiles
+      Array(@cover_tempfiles).each(&:close!)
+    end
+
+    ## The element's cover selection (chem-generic-ui#1050) rendered into the
+    ## export: the stored `{source, id}` references are resolved through the
+    ## same helpers as the cover_image API, so the docx shows exactly what the
+    ## element page shows. Only rasters Word renders reliably are embedded as
+    ## originals; everything else falls back to the PNG thumbnail.
+    DOCX_COVER_MIMES = %w[image/png image/jpeg image/gif image/bmp].freeze
+    DOCX_COVER_EXT = {
+      'image/png' => '.png',
+      'image/jpeg' => '.jpg',
+      'image/gif' => '.gif',
+      'image/bmp' => '.bmp'
+    }.freeze
+    # px-unit bounds of the rendered image (Sablon::Image::PX_TO_EMU scale):
+    # 1600 ≈ 13.3cm wide, 1100 ≈ 9.2cm tall
+    DOCX_COVER_MAX_W = 1600
+    DOCX_COVER_MAX_H = 1100
+
+    def cover_context
+      covers = build_cover_images
+      { has_cover: covers.present?, cover_images: covers }
+    end
+
+    def build_cover_images
+      return [] unless @element.instance_of?(Labimotion::Element)
+
+      refs = @element.metadata&.fetch('cover_images', nil) || []
+      refs.filter_map { |ref| build_cover_image(ref) }
     rescue StandardError => e
       Labimotion.log_exception(e)
-    ensure
-      # Close and delete the temporary file
-      tempfile&.close
-      tempfile&.unlink
+      []
+    end
+
+    def build_cover_image(ref)
+      att = cover_helpers.cover_image_attachment(@element, ref['source'], ref['id'])
+      data, mime = att && docx_cover_payload(att)
+      return nil if data.nil?
+
+      { image: docx_cover_sablon_image(data, mime), caption: docx_cover_caption(ref, att) }
+    rescue StandardError => e
+      Labimotion.log_exception(e)
+      nil
+    end
+
+    def docx_cover_payload(att)
+      original = docx_cover_original(att)
+      return original if original
+
+      thumb = att.read_thumbnail
+      [thumb, 'image/png'] if thumb
+    end
+
+    def docx_cover_original(att)
+      return nil unless att.type_image? && !att.type_image_tiff?
+
+      mime = att.content_type.to_s.downcase
+      data = att.read_file
+      return nil if data.nil?
+      return [data, mime] if DOCX_COVER_MIMES.include?(mime)
+
+      # the template's blip placeholder takes bitmaps only, so svg & friends
+      # are rasterized onto white through the shared cover helper
+      converted = cover_helpers.cover_rasterize(data, mime)
+      [converted, 'image/png'] if converted
+    end
+
+    def docx_cover_sablon_image(data, mime)
+      tmp = Tempfile.new(['labimotion-cover', DOCX_COVER_EXT.fetch(mime, '.png')])
+      tmp.binmode
+      tmp.write(data)
+      tmp.flush
+      @cover_tempfiles << tmp
+      image = Sablon::Image.create_by_path(tmp.path)
+      apply_docx_cover_dimensions(image, data)
+      image
+    end
+
+    ## Without dimensions the placeholder geometry of the template is kept,
+    ## which distorts the aspect ratio — best effort, never fatal.
+    def apply_docx_cover_dimensions(image, data)
+      require 'mini_magick'
+      width, height = MiniMagick::Image.read(data).dimensions
+      return unless width.to_i.positive? && height.to_i.positive?
+
+      scale = [1.0, DOCX_COVER_MAX_W.to_f / width, DOCX_COVER_MAX_H.to_f / height].min
+      image.px_width = (width * scale).round
+      image.px_height = (height * scale).round
+    rescue LoadError, StandardError => e
+      Labimotion.log_exception(e)
+    end
+
+    def docx_cover_caption(ref, att)
+      label = ref['source'] == 'analysis' ? @element.analyses.find_by(id: ref['id'])&.name : att.filename
+      "ref: #{ref['source']}: #{label}"
+    end
+
+    def cover_helpers
+      @cover_helpers ||= Object.new.extend(Labimotion::CoverImageHelpers)
     end
 
 

data/lib/labimotion/libs/file_extractor.rb

@@ -0,0 +1,210 @@
+# frozen_string_literal: true
+
+require 'base64'
+require 'stringio'
+require 'timeout'
+require 'tempfile'
+require 'date'
+
+# Optional parsers — the feature degrades to filename-only if a gem is absent.
+begin
+  require 'pdf/reader'
+rescue LoadError
+  nil
+end
+begin
+  require 'roo'
+rescue LoadError
+  nil
+end
+
+module Labimotion
+  # FileExtractor turns an uploaded file (base64-encoded raw bytes + filename)
+  # into plain text usable as LLM context for the AI dataset-template feature.
+  #
+  # It NEVER raises into the caller. Any failure — missing gem, unsupported or
+  # oversized or corrupt file, parse error, timeout, out-of-memory — returns
+  # either '' or a short bracketed marker describing why, so the request keeps
+  # working with filename-only context.
+  module FileExtractor
+    TEXT_EXTS = %w[txt csv tsv md markdown json xml yml yaml].freeze
+    BINARY_EXTS = %w[pdf xlsx xlsm xls].freeze
+    ALLOWED_EXTS = (TEXT_EXTS + BINARY_EXTS).freeze
+
+    MAX_DECODED_BYTES = 15 * 1024 * 1024   # reject decoded payloads larger than this
+    MAX_UNCOMPRESSED = 80 * 1024 * 1024    # xlsx zip-bomb guard (sum of entry sizes)
+    MAX_OUT_CHARS = 8000                   # per-file extracted-text cap
+    MAX_PDF_PAGES = 50
+    MAX_SHEETS = 5
+    MAX_ROWS = 100
+    MAX_COLS = 30
+    MAX_CELL = 200
+    EXTRACT_TIMEOUT = 20                   # best-effort (cannot interrupt C-ext parsing)
+
+    # Magic bytes — forced to binary so start_with? never hits an
+    # Encoding::CompatibilityError against the ASCII-8BIT decoded bytes.
+    ZIP_MAGIC = "PK\x03\x04".b
+    OLE2_MAGIC = "\xD0\xCF\x11\xE0".b
+    PDF_MAGIC = '%PDF'.b
+
+    class << self
+      # @param filename [String] used for the extension whitelist + dispatch
+      # @param content_base64 [String] base64 of the raw file bytes (data-URL prefix tolerated)
+      # @return [String] extracted text, a '[...]' marker, or ''
+      def extract(filename, content_base64)
+        extract_bytes(filename, decode(content_base64))
+      end
+
+      # Same as #extract but for already-decoded RAW bytes (e.g. Attachment#read_file).
+      # Keeps every guard #extract has: extension whitelist, empty check,
+      # MAX_DECODED_BYTES cap, the extraction Timeout, output truncation and the
+      # four rescue clauses. NEVER raises into the caller.
+      #
+      # @param filename [String] used for the extension whitelist + dispatch
+      # @param bytes [String, nil] raw file bytes (any encoding; treated as binary)
+      # @return [String] extracted text, a '[...]' marker, or ''
+      def extract_bytes(filename, bytes)
+        ext = File.extname(filename.to_s).downcase.delete('.')
+        return '' unless ALLOWED_EXTS.include?(ext)
+
+        return '' if bytes.nil? || bytes.empty?
+        return '[file too large to extract]' if bytes.bytesize > MAX_DECODED_BYTES
+
+        truncate(Timeout.timeout(EXTRACT_TIMEOUT) { dispatch(ext, bytes) })
+      rescue Timeout::Error
+        '[extraction timed out]'
+      rescue NoMemoryError, SystemStackError => e
+        log(e)
+        '[file too complex to extract]'
+      rescue StandardError => e
+        log(e)
+        ''
+      end
+
+      private
+
+      def decode(content_base64)
+        str = content_base64.to_s.sub(/\Adata:[^;,]*;base64,/, '') # tolerate data-URL prefixes
+        return nil if str.strip.empty?
+
+        Base64.decode64(str)
+      end
+
+      # Dispatch by magic bytes first (robust against spoofed/missing extensions),
+      # falling back to the extension.
+      def dispatch(ext, bytes)
+        head = bytes[0, 8].to_s.b
+        if head.start_with?(ZIP_MAGIC) || %w[xlsx xlsm].include?(ext)
+          extract_xlsx(bytes)
+        elsif head.start_with?(OLE2_MAGIC) || ext == 'xls'
+          extract_xls(bytes)
+        elsif head.start_with?(PDF_MAGIC) || ext == 'pdf'
+          extract_pdf(bytes)
+        else
+          extract_text(bytes)
+        end
+      end
+
+      def extract_pdf(bytes)
+        return '[PDF support unavailable on this server]' unless defined?(PDF::Reader)
+
+        reader = PDF::Reader.new(StringIO.new(bytes))
+        buf = +''
+        reader.pages.first(MAX_PDF_PAGES).each do |page|
+          buf << page.text.to_s << "\n\n"
+          break if buf.length >= MAX_OUT_CHARS
+        end
+        text = buf.scrub('').strip
+        text.empty? ? '[no extractable text - likely a scanned/image PDF]' : text
+      rescue StandardError => e
+        log(e)
+        e.class.name.to_s.include?('Encrypted') ? '[encrypted PDF - no text extracted]' : ''
+      end
+
+      def extract_xlsx(bytes)
+        return '[Excel support unavailable on this server]' unless defined?(Roo::Excelx)
+        return '[Excel file too large to extract]' if xlsx_uncompressed_too_big?(bytes)
+
+        render_workbook(Roo::Excelx.new(StringIO.new(bytes)))
+      end
+
+      # Legacy OLE2 .xls needs the roo-xls gem (not bundled here) and a real path.
+      def extract_xls(bytes)
+        return '[legacy .xls not supported on this server]' unless defined?(Roo::Excel)
+
+        Tempfile.create(['labimotion-upload', '.xls']) do |f|
+          f.binmode
+          f.write(bytes)
+          f.flush
+          render_workbook(Roo::Spreadsheet.open(f.path))
+        end
+      end
+
+      def extract_text(bytes)
+        bytes.dup.force_encoding('UTF-8').scrub('')[0, MAX_OUT_CHARS].to_s
+      end
+
+      # Zip-bomb guard: sum the uncompressed entry sizes before roo inflates the
+      # archive. roo bundles rubyzip, so Zip is available when roo loaded.
+      def xlsx_uncompressed_too_big?(bytes)
+        return false unless defined?(Zip)
+
+        total = 0
+        Zip::File.open_buffer(StringIO.new(bytes)) do |zip|
+          zip.each do |entry|
+            total += entry.size.to_i
+            return true if total > MAX_UNCOMPRESSED
+          end
+        end
+        false
+      rescue StandardError
+        false # if we can't pre-inspect, let roo try (still bounded by caps + timeout)
+      end
+
+      # Render up to MAX_SHEETS sheets, each with a "## <name>" header and a
+      # capped, pipe-delimited row dump, under a shared output-char budget.
+      def render_workbook(book)
+        buf = +''
+        names = Array(book.sheets)
+        names.first(MAX_SHEETS).each do |name|
+          sheet = book.sheet(name)
+          next unless sheet.last_row # skip empty sheets
+
+          buf << "## #{name}\n"
+          last = [sheet.last_row, sheet.first_row + MAX_ROWS - 1].min
+          (sheet.first_row..last).each do |i|
+            line = +''
+            line << sheet.row(i).first(MAX_COLS).map { |c| fmt_cell(c) }.join(' | ') << "\n"
+            break if buf.length + line.length > MAX_OUT_CHARS
+
+            buf << line
+          end
+          buf << "[+#{sheet.last_row - last} more rows]\n" if sheet.last_row > last
+          break if buf.length >= MAX_OUT_CHARS
+        end
+        buf << "[+#{names.size - MAX_SHEETS} more sheets omitted]\n" if names.size > MAX_SHEETS
+        buf
+      end
+
+      # roo returns Date/Time/DateTime objects and nil for empty cells.
+      def fmt_cell(cell)
+        case cell
+        when nil then ''
+        when Date, Time, DateTime then cell.iso8601
+        else cell.to_s[0, MAX_CELL]
+        end
+      end
+
+      def truncate(str)
+        s = str.to_s
+        s.length > MAX_OUT_CHARS ? "#{s[0, MAX_OUT_CHARS]}\n…[truncated]" : s
+      end
+
+      def log(err)
+        Labimotion.log_exception(err) if Labimotion.respond_to?(:log_exception)
+      rescue StandardError
+        nil
+      end
+    end
+  end
+end