labimotion 2.4.0.rc2 → 2.4.0.rc4

data/lib/labimotion/libs/ai_template.rb

@@ -0,0 +1,1482 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'securerandom'
+require 'labimotion/utils/units'
+
+module Labimotion
+  # AiTemplate generates a generic dataset template (DatasetKlass properties
+  # template) from a natural-language description using an LLM.
+  #
+  # The client targets the KIT KI-Toolbox (https://ki-toolbox.scc.kit.edu), an
+  # OpenAI-compatible chat-completions gateway (Open WebUI), but is fully
+  # config-driven (config/labimotion_ai.yml, then ENV) so it can point at any
+  # compatible service. Settings resolve yml -> ENV -> default:
+  #
+  #   :api_key  / KI_TOOLBOX_API_KEY    (required) bearer token (created in the UI)
+  #   :base_url / KI_TOOLBOX_BASE_URL   default https://ki-toolbox.scc.kit.edu
+  #   :api_path / KI_TOOLBOX_API_PATH   default /api/v1/chat/completions
+  #   :model    / KI_TOOLBOX_MODEL      default azure.gpt-4.1-mini
+  #   :max_tokens, :timeout, :system_prompt
+  #
+  # The LLM is asked to return a JSON document describing the template layers
+  # and fields in the LabIMotion format. Only the data part is returned here;
+  # wrapping it into a persistable DatasetKlass (uuid, pkg, klass, ...) is the
+  # caller's job (see Labimotion::DatasetHelpers#create_ai_dataset_klass).
+  class AiTemplate
+    DEFAULT_BASE_URL = 'https://ki-toolbox.scc.kit.edu'
+    DEFAULT_API_PATH = '/api/v1/chat/completions'
+    DEFAULT_MODEL = 'azure.gpt-4.1-mini'
+    DEFAULT_MAX_TOKENS = 8000
+    # A connection test only needs a valid round-trip, not a useful completion, so
+    # cap the reply hard to keep the ping cheap and fast.
+    PING_MAX_TOKENS = 8
+    REQUEST_TIMEOUT = 120
+    MAX_FILES = 10
+    # Slightly above FileExtractor::MAX_OUT_CHARS (8000) so its "[truncated]"
+    # marker is not chopped off by the slice below.
+    MAX_FILE_CHARS = 9000
+
+    # Field "type" values allowed in generated templates — a practical subset of
+    # the LabIMotion field-base schema enum.
+    FIELD_TYPES = %w[
+      text textarea integer number checkbox date datetime
+      select select-multi ontology-select system-defined label upload table
+    ].freeze
+
+    # Field types whose choices live in the shared select_options map.
+    SELECT_TYPES = %w[select select-multi].freeze
+
+    # Field types that may carry a physical unit (mapped to a system-defined group).
+    UNIT_CANDIDATE_TYPES = %w[number integer system-defined].freeze
+
+    # Column types allowed inside a "table" field's sub_fields; others -> text.
+    TABLE_COL_TYPES = %w[text number integer checkbox date datetime].freeze
+
+    # Generate a metadata template for a generic dataset, element or segment.
+    #
+    # The JSON template schema (layers / fields / select_options) is IDENTICAL
+    # across all three kinds — only the prompt wording changes. The dataset path
+    # is driven by a CHMO ontology term; the element/segment paths are driven by
+    # a subject string (the element/segment name/label).
+    #
+    # @param kind [String] 'dataset' (default), 'element' or 'segment'
+    # @param ols_term_id [String] CHMO term id, e.g. "CHMO:0000470 | mass spectrometry (MS)" (dataset only)
+    # @param subject [String] the element/segment name/label (element/segment only)
+    # @param desc [String] free-text description of the template
+    # @param cols [Integer] default columns per row for generated layers (1-6)
+    # @param references [Array<String>] reference links (publications / standards)
+    # @param files [Array<Hash>] [{ 'filename' => ..., 'content' => ... }, ...]
+    # @return [Hash] { 'label' => String, 'layers' => Hash, 'select_options' => Hash }
+    def self.generate(kind: 'dataset', ols_term_id: nil, subject: nil, desc: nil, cols: 1, references: [], files: [], model: nil, api_key: nil, base_url: nil, api_path: nil)
+      new(kind: kind, ols_term_id: ols_term_id, subject: subject, desc: desc, cols: cols, references: references,
+          files: files, model: model, api_key: api_key, base_url: base_url, api_path: api_path).generate
+    end
+
+    # Refine an existing dataset template through a natural-language instruction.
+    #
+    # The caller passes the CURRENT template (label, layers, select_options) —
+    # which already reflects any previously-applied refinements — plus the latest
+    # instruction and an optional short history of prior turns for continuity.
+    # The LLM returns the FULL revised template in the same shape, together with
+    # a one-line summary of what it changed. Nothing is persisted here; the admin
+    # reviews the result in the designer and saves it (human-in-the-loop).
+    #
+    # @param current [Hash] { 'label' => String, 'layers' => Hash, 'select_options' => Hash }
+    # @param instruction [String] the change to apply
+    # @param history [Array<Hash>] prior turns [{ 'role' => 'user'|'assistant', 'content' => String }]
+    # @param cols [Integer] fallback columns per row when a layer omits its own (1-6)
+    # @param model [String, nil] per-user model override (falls back to yml/ENV/default)
+    # @param api_key [String, nil] per-user API key override (falls back to yml/ENV)
+    # @param base_url [String, nil] per-user provider base URL (honored only with api_key)
+    # @param api_path [String, nil] per-user provider chat-completions path (honored only with api_key)
+    # @return [Hash] { 'label' => String, 'layers' => Hash, 'select_options' => Hash, 'summary' => String }
+    def self.refine(current:, instruction:, ols_term_id: nil, history: [], cols: 1, model: nil, api_key: nil, base_url: nil, api_path: nil)
+      new(ols_term_id: ols_term_id, cols: cols, history: history, model: model, api_key: api_key, base_url: base_url, api_path: api_path)
+        .refine(current: current, instruction: instruction)
+    end
+
+    # Extract DATA VALUES for an existing generic element/segment/dataset instance
+    # from a document's text, to pre-fill the working copy for human review. This
+    # does NOT design or modify a template — it only reads values for the fields the
+    # given `properties` template already defines. Nothing is persisted here.
+    #
+    # @param properties [Hash] the element instance properties ({ 'layers' => ..., 'select_options' => ... })
+    # @param context_text [String] plain text extracted from the source document(s)
+    # @param model [String, nil] per-user model override (falls back to yml/ENV/default)
+    # @param api_key [String, nil] per-user API key override (falls back to yml/ENV)
+    # @param base_url [String, nil] per-user provider base URL (honored only with api_key)
+    # @param api_path [String, nil] per-user provider chat-completions path (honored only with api_key)
+    # @return [Hash] { 'values' => { layer_key => { field_key => value } }, 'summary' => String }
+    def self.fill(properties:, context_text:, model: nil, api_key: nil, base_url: nil, api_path: nil)
+      new(ols_term_id: '', model: model, api_key: api_key, base_url: base_url, api_path: api_path)
+        .fill(properties: properties, context_text: context_text)
+    end
+
+    # Verify the (per-user or server) AI settings by sending one tiny chat request
+    # and reporting whether it round-trips. Nothing is generated or persisted. Same
+    # resolution as the real calls: base_url/api_path are honored only ALONGSIDE a
+    # personal key (custom_endpoint?), and a custom base_url is SSRF-validated.
+    #
+    # @return [Hash] { 'ok' => true, 'model' => .., 'endpoint' => .., 'ms' => .. }
+    # @raise [RuntimeError] with a user-facing message on any failure
+    def self.ping(model: nil, api_key: nil, base_url: nil, api_path: nil)
+      new(ols_term_id: '', model: model, api_key: api_key, base_url: base_url, api_path: api_path).ping
+    end
+
+    # Reverse index of physical units -> the LabIMotion system-defined unit group,
+    # built once from Labimotion::Units::FIELDS. Maps a normalized unit token (from
+    # a unit's label OR its key) to { group:, value_system: }. Tokens used by more
+    # than one group are AMBIGUOUS and dropped, so a lookup never guesses the wrong
+    # group (e.g. "Pa" -> pressure vs elastic modulus falls back to a plain number).
+    def self.unit_index
+      @unit_index ||= build_unit_index
+    end
+
+    # All valid unit-group keys (the option_layers value of a system-defined field).
+    def self.unit_groups
+      @unit_groups ||= Labimotion::Units::FIELDS.map { |g| g[:field].to_s }.freeze
+    end
+
+    def self.build_unit_index
+      collect_unit_tokens.each_with_object({}) do |(token, groups), index|
+        next unless groups.size == 1
+
+        gkey, ukey = groups.first
+        index[token] = { group: gkey, value_system: ukey }
+      end
+    end
+
+    # token -> { group => first_unit_key }. A token seen under >1 group signals it
+    # is ambiguous (kept here, filtered out in build_unit_index).
+    def self.collect_unit_tokens
+      tokens = {}
+      Labimotion::Units::FIELDS.each do |group|
+        gkey = group[:field].to_s
+        Array(group[:units]).each { |unit| record_unit_tokens(tokens, gkey, unit) }
+      end
+      tokens
+    end
+
+    def self.record_unit_tokens(tokens, gkey, unit)
+      ukey = unit[:key].to_s
+      return if ukey.empty?
+
+      [unit[:label], ukey].each do |raw|
+        token = normalize_unit_token(raw)
+        next if token.empty?
+
+        (tokens[token] ||= {})[gkey] ||= ukey
+      end
+    end
+
+    # Fold a unit string to a comparable token: strip HTML (<sup>2</sup> -> 2), map
+    # unicode superscripts to digits, micro sign to "u", downcase, drop spaces. So
+    # "°C", "µmol/L", "g/cm<sup>3</sup>" match the model's "°c"/"umol/l"/"g/cm3".
+    def self.normalize_unit_token(str)
+      s = str.to_s.gsub(/<[^>]+>/, '')
+      s = s.tr('²³¹⁰⁴⁵⁶⁷⁸⁹', '2310456789')
+      s = s.gsub(/[µμ]/, 'u').downcase
+      s.gsub(/\s+/, '')
+    end
+
+    def initialize(kind: 'dataset', ols_term_id: nil, subject: nil, desc: nil, cols: 1, references: [], files: [], history: [], model: nil, api_key: nil, base_url: nil, api_path: nil)
+      @kind = kind.to_s
+      @subject = subject.to_s.strip
+      @ols_term_id = ols_term_id.to_s.strip
+      @desc = desc.to_s.strip
+      @cols = cols.to_i.clamp(1, 6)
+      @references = Array(references).map(&:to_s).reject(&:blank?)
+      @files = Array(files)
+      @history = Array(history)
+      # Per-user overrides (from the caller); blank -> fall back to yml/ENV/default.
+      @model_override = model.to_s.strip
+      @api_key_override = api_key.to_s.strip
+      # A user may only redirect to their OWN provider when they also bring their
+      # own key — the shared server key must never be sent to a user-supplied URL.
+      @base_url_override = base_url.to_s.strip
+      @api_path_override = api_path.to_s.strip
+    end
+
+    def generate
+      raise 'AI API key is not configured (set config/labimotion_ai.yml :api_key or KI_TOOLBOX_API_KEY)' if api_key.blank?
+      if @kind == 'dataset'
+        raise 'An ontology term (CHMO) is required' if @ols_term_id.blank?
+      else
+        raise 'A name/label is required' if @subject.blank?
+      end
+
+      response = post_messages
+      raise "AI request failed (HTTP #{response.code})" unless response.code == 200
+
+      body = JSON.parse(response.body)
+      guard_finish_reason!(body)
+
+      text = extract_text(body)
+      raise 'AI returned an empty response' if text.blank?
+
+      normalize(parse_json(text))
+    rescue JSON::ParserError => e
+      log_ai_response('generate could not parse response', response&.body)
+      Labimotion.log_exception(e)
+      raise 'AI returned a response that could not be parsed as a template'
+    end
+
+    def refine(current:, instruction:)
+      raise 'AI API key is not configured (set config/labimotion_ai.yml :api_key or KI_TOOLBOX_API_KEY)' if api_key.blank?
+      raise 'An instruction is required' if instruction.to_s.strip.blank?
+
+      response = post_chat(refine_messages(current, instruction))
+      raise "AI request failed (HTTP #{response.code})" unless response.code == 200
+
+      body = JSON.parse(response.body)
+      guard_finish_reason!(body)
+
+      text = extract_text(body)
+      raise 'AI returned an empty response' if text.blank?
+
+      data = parse_json(text)
+      result = normalize_preserving(data)
+      # Guard against a malformed response silently wiping the template — a
+      # dataset template always has at least one layer.
+      if result['layers'].empty?
+        log_ai_response('refine produced no layers', text)
+        raise 'AI returned an empty template (the model response contained no layers). ' \
+              'Try a more capable model.'
+      end
+
+      result['summary'] = (data.is_a?(Hash) ? data['summary'].to_s.strip : '')
+      result
+    rescue JSON::ParserError => e
+      log_ai_response('refine could not parse response', response&.body)
+      Labimotion.log_exception(e)
+      raise 'AI returned a response that could not be parsed as a template'
+    end
+
+    def fill(properties:, context_text:)
+      raise 'AI API key is not configured (set config/labimotion_ai.yml :api_key or KI_TOOLBOX_API_KEY)' if api_key.blank?
+      raise 'No readable text could be extracted from the selected source' if context_text.to_s.strip.blank?
+
+      props = properties.is_a?(Hash) ? properties : {}
+      schema = fill_field_schema(props)
+      raise 'This element template has no fields to fill' if schema.empty?
+
+      response = post_chat(fill_messages(schema, context_text))
+      raise "AI request failed (HTTP #{response.code})" unless response.code == 200
+
+      body = JSON.parse(response.body)
+      guard_finish_reason!(body)
+
+      text = extract_text(body)
+      raise 'AI returned an empty response' if text.blank?
+
+      data = parse_json(text)
+      raw_values = data.is_a?(Hash) ? data['values'] : nil
+      {
+        'values' => normalize_fill_values(props, raw_values),
+        'summary' => (data.is_a?(Hash) ? data['summary'].to_s.strip : '')
+      }
+    rescue JSON::ParserError => e
+      log_ai_response('fill could not parse response', response&.body)
+      Labimotion.log_exception(e)
+      raise 'AI returned a response that could not be parsed'
+    end
+
+    def ping
+      raise 'AI API key is not configured' if api_key.blank?
+
+      started  = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      response = post_chat([{ role: 'user', content: 'ping' }], PING_MAX_TOKENS)
+      ms       = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - started) * 1000).round
+      raise ping_error(response.code) unless response.code == 200
+
+      { 'ok' => true, 'model' => model, 'endpoint' => "#{base_url}#{api_path}", 'ms' => ms }
+    end
+
+    private
+
+    # Raise a clear error for terminal finish_reasons BEFORE we try to parse: a
+    # content-filter block, or a response truncated at the token limit — the latter
+    # would otherwise surface as a confusing parse/empty-template error.
+    def guard_finish_reason!(body)
+      case Array(body['choices']).first&.dig('finish_reason')
+      when 'content_filter'
+        raise 'AI request was declined by the content filter'
+      when 'length'
+        raise 'AI response was cut off at the token limit — increase :max_tokens in ' \
+              'config/labimotion_ai.yml or choose a model with a larger output limit.'
+      end
+    end
+
+    # Best-effort: record the raw model output (truncated) to log/labimotion.log so a
+    # response that can't be turned into a template can be diagnosed. Never raises.
+    def log_ai_response(context, raw)
+      return if raw.to_s.strip.empty?
+
+      Labimotion.logger.warn("AiTemplate #{context}; raw (truncated): #{raw.to_s[0, 2000]}")
+    rescue StandardError
+      nil
+    end
+
+    # Map a non-200 status from the ping into a message that points at the setting
+    # most likely at fault. Network/SSRF failures raise before this (from post_chat).
+    def ping_error(code)
+      case code.to_i
+      when 401, 403 then 'Authentication failed — check the API key.'
+      when 404      then 'Endpoint not found — check the Provider URL and path.'
+      when 400, 422 then 'Provider rejected the request — check the model id.'
+      when 300..399 then 'Endpoint redirected — not allowed.'
+      else "AI request failed (HTTP #{code})."
+      end
+    end
+
+    # config/labimotion_ai.yml (Rails.configuration.labimotion_ai) for the
+    # current environment, or {} when the app/initializer is unavailable.
+    def cfg
+      @cfg ||= begin
+        loaded = (Rails.configuration.labimotion_ai if defined?(Rails) && Rails.respond_to?(:configuration))
+        loaded || {}
+      rescue StandardError
+        {}
+      end
+    end
+
+    # Resolve a setting: yml value first, then ENV var, then the built-in default.
+    def setting(key, env_key, default = nil)
+      cfg[key].presence || ENV[env_key].presence || default
+    end
+
+    def api_key
+      @api_key ||= (@api_key_override.presence || setting(:api_key, 'KI_TOOLBOX_API_KEY', nil))
+    end
+
+    # True when the caller brought BOTH a personal key and a provider URL: only
+    # then is the user-supplied endpoint honored (the server key is never sent to
+    # a user URL). A custom endpoint is SSRF-validated before use.
+    def custom_endpoint?
+      @api_key_override.present? && @base_url_override.present?
+    end
+
+    def base_url
+      @base_url ||=
+        if custom_endpoint?
+          Labimotion::AiEgressGuard.validate!(@base_url_override)
+        else
+          setting(:base_url, 'KI_TOOLBOX_BASE_URL', DEFAULT_BASE_URL)
+        end.to_s.chomp('/')
+    end
+
+    def api_path
+      return @api_path_override if custom_endpoint? && @api_path_override.present?
+
+      setting(:api_path, 'KI_TOOLBOX_API_PATH', DEFAULT_API_PATH)
+    end
+
+    def model
+      @model_override.presence || setting(:model, 'KI_TOOLBOX_MODEL', DEFAULT_MODEL)
+    end
+
+    def max_tokens
+      setting(:max_tokens, 'KI_TOOLBOX_MAX_TOKENS', DEFAULT_MAX_TOKENS).to_i
+    end
+
+    def request_timeout
+      setting(:timeout, 'KI_TOOLBOX_TIMEOUT', REQUEST_TIMEOUT).to_i
+    end
+
+    def post_messages
+      post_chat(
+        [
+          { role: 'system', content: [system_prompt, field_rules_prompt].join("\n\n") },
+          { role: 'user', content: user_prompt }
+        ]
+      )
+    end
+
+    # POST an OpenAI-compatible chat-completions request for the given messages.
+    # tokens defaults to the full max_tokens; the connection ping passes a small
+    # cap so a test round-trip stays cheap.
+    def post_chat(messages, tokens = max_tokens)
+      HTTParty.post(
+        "#{base_url}#{api_path}",
+        headers: {
+          'Authorization' => "Bearer #{api_key}",
+          'Content-Type' => 'application/json'
+        },
+        body: {
+          model: model,
+          max_tokens: tokens,
+          messages: messages
+        }.to_json,
+        timeout: request_timeout,
+        # Never follow redirects: a public URL that 302s to an internal address
+        # is the classic SSRF bypass of the egress guard.
+        follow_redirects: false
+      )
+    end
+
+    # OpenAI-compatible chat completions return the text at
+    # choices[0].message.content.
+    def extract_text(body)
+      choice = Array(body['choices']).first || {}
+      (choice['message'] || {})['content'].to_s.strip
+    end
+
+    # The system prompt comes from config/labimotion_ai.yml (kind-specific key);
+    # the default_*_system_prompt is the built-in fallback for each kind.
+    def system_prompt
+      case @kind
+      when 'element'
+        cfg[:element_system_prompt].presence || default_element_system_prompt
+      when 'segment'
+        cfg[:segment_system_prompt].presence || default_segment_system_prompt
+      else
+        cfg[:system_prompt].presence || default_system_prompt
+      end
+    end
+
+    def default_system_prompt
+      <<~PROMPT
+        You are an assistant that designs metadata templates for analytical
+        chemistry datasets in the Chemotion ELN (LabIMotion generic dataset
+        framework).
+
+        Given an analytical method (a CHMO ontology term) and optional context,
+        design a concise, practical dataset template that a chemist would use to
+        record the metadata and results of that kind of measurement.
+
+        Respond with a SINGLE JSON object only — no prose, no markdown, no code
+        fences. The JSON must have exactly this shape:
+
+        {
+          "label": "<short human-readable template name>",
+          "layers": {
+            "<layer_key>": {
+              "key": "<layer_key>",
+              "label": "<Layer label>",
+              "cols": 1,
+              "color": "default",
+              "position": 1,
+              "group": "<optional UI section name>",
+              "fields": [
+                {
+                  "field": "<field_key>",
+                  "type": "text",
+                  "label": "<Field label>",
+                  "position": 1,
+                  "required": false
+                }
+              ]
+            }
+          },
+          "select_options": {}
+        }
+
+        Rules (LabIMotion property schema):
+        - layer_key and field_key are lowercase snake_case, unique within their
+          scope, and contain only [a-z0-9_]. A field "label" is human-readable and
+          keeps standard acronyms (IUPAC, SMILES, m/z).
+        - Each layer MUST include: key, label, cols (integer, usually 1),
+          color ("default"), position (1-based integer) and fields (array).
+        - Each field MUST include: field, type, label, position, required.
+        - "type" MUST be one of: "text", "textarea", "integer", "number",
+          "checkbox", "date", "datetime", "select", "select-multi",
+          "ontology-select", "system-defined", "upload", "table".
+        - For a "select"/"select-multi" field, list the choices INLINE on the field
+          as "options": [ { "key": "k", "label": "Label" } ] (a plain string array
+          also works); do not set option_layers or a separate select_options block.
+        Design guidance — recognise the method and build the layers IT needs:
+        - Organise fields into layers that follow the workflow of the technique,
+          drawing from: an "overview" layer (operator, measurement date, and a
+          "method" field of type "ontology-select" whose default is the given
+          ontology term); an instrument/setup layer; an acquisition/conditions layer
+          (parameters that define how the measurement is run); a sample layer; a
+          processing layer; and a results layer. Include ONLY the layers that fit
+          this method and add method-specific ones. Aim for practical coverage
+          (about 3-8 layers), not exhaustiveness.
+        - Do NOT create a "general" or "General description" layer — the system adds
+          that (title, description). Put administrative fields in "overview".
+        - Optionally set a layer "group" (a short section name, e.g. "Measurement",
+          "Sample & data") to bundle related layers into one UI section; leave
+          "overview" ungrouped. The system builds the section index from these tags.
+        - Keep it method-specific; do not duplicate fields already implied by the
+          analysis container (e.g. the analysis kind/type itself).
+      PROMPT
+    end
+
+    def default_element_system_prompt
+      generic_system_prompt('element')
+    end
+
+    def default_segment_system_prompt
+      generic_system_prompt('segment')
+    end
+
+    # System prompt for the element/segment kinds. The JSON-shape rules are
+    # identical to the dataset prompt — only the noun ("element"/"segment") and
+    # the framing sentence differ.
+    def generic_system_prompt(noun)
+      <<~PROMPT
+        You are an assistant that designs metadata templates for a LabIMotion
+        generic #{noun} in the Chemotion ELN (LabIMotion generic element
+        framework).
+
+        Given a #{noun} name/label and optional context, design a concise,
+        practical metadata template that a scientist would use to record the
+        structured metadata for that kind of #{noun}.
+
+        Respond with a SINGLE JSON object only — no prose, no markdown, no code
+        fences. The JSON must have exactly this shape:
+
+        {
+          "label": "<short human-readable template name>",
+          "layers": {
+            "<layer_key>": {
+              "key": "<layer_key>",
+              "label": "<Layer label>",
+              "cols": 1,
+              "color": "default",
+              "position": 1,
+              "group": "<optional UI section name>",
+              "fields": [
+                {
+                  "field": "<field_key>",
+                  "type": "text",
+                  "label": "<Field label>",
+                  "position": 1,
+                  "required": false
+                }
+              ]
+            }
+          },
+          "select_options": {}
+        }
+
+        Rules (LabIMotion property schema):
+        - layer_key and field_key are lowercase snake_case, unique within their
+          scope, and contain only [a-z0-9_]. A field "label" is human-readable and
+          keeps standard acronyms (IUPAC, SMILES, m/z).
+        - Each layer MUST include: key, label, cols (integer, usually 1),
+          color ("default"), position (1-based integer) and fields (array).
+        - Each field MUST include: field, type, label, position, required.
+        - "type" MUST be one of: "text", "textarea", "integer", "number",
+          "checkbox", "date", "datetime", "select", "select-multi",
+          "ontology-select", "system-defined", "upload", "table".
+        - For a "select"/"select-multi" field, list the choices INLINE on the field
+          as "options": [ { "key": "k", "label": "Label" } ] (a plain string array
+          also works); do not set option_layers or a separate select_options block.
+        Design guidance — recognise the #{noun} and build the layers IT needs:
+        - Organise fields into layers by theme, drawing from: an "overview" layer
+          (identifiers, source/origin, responsible person, date); one or more
+          property/attribute layers describing the #{noun}; measurement or result
+          layers; and a notes layer. Include ONLY the layers that fit and add
+          #{noun}-specific ones. Aim for practical coverage (about 3-8 layers), not
+          exhaustiveness.
+        - Do NOT create a "general" or "General description" layer — the system adds
+          that. Put administrative fields in "overview".
+        - Optionally set a layer "group" (a short section name) to bundle related
+          layers into one UI section; leave "overview" ungrouped. The system builds
+          the section index from these tags.
+        - Keep it specific to the #{noun}; do not duplicate fields already implied by
+          the container.
+      PROMPT
+    end
+
+    # Extra field-type rules appended to EVERY system prompt (create and refine,
+    # all kinds), independent of the customizable system prompt so they always
+    # reach the model. Two rules the models otherwise get wrong: numbers that
+    # carry a unit must be "system-defined" (with a unit group), and fixed-choice
+    # values must be "select"/"select-multi" (with matching select_options). The
+    # unit-group keys are the ONLY valid option_layers for system-defined fields
+    # (a curated subset of Labimotion::Units::FIELDS).
+    def field_rules_prompt
+      <<~PROMPT
+        Additional field-type rules — apply these when choosing each field's "type":
+
+        1) A NUMBER THAT HAS A PHYSICAL UNIT (temperature, mass, volume, amount,
+           concentration, pressure, time, length, energy, voltage, current, frequency,
+           speed, percentage, etc.) must carry its unit. Use type "number" and add a
+           "unit" field with the unit symbol, e.g.:
+             { "field": "temp", "type": "number", "label": "Temperature", "unit": "°C" }
+           More unit examples: "mg", "mL", "µmol/L", "bar", "kJ/mol", "Hz", "%", "V".
+           The system maps the unit to the correct unit group automatically.
+           If the unit is NOT a standard physical unit (e.g. m/z, cm⁻¹, lines/mm,
+           a.u.) or the number has no unit, use "number" with NO "unit" and put any
+           unit in the LABEL instead, e.g. "Spectral resolution (cm⁻¹)".
+
+        2) A VALUE CHOSEN FROM A FIXED SET OF OPTIONS must use "select" (one choice)
+           or "select-multi" (several at once) — never a free-text "text" field. List
+           the choices INLINE on the field:
+             "options": [ { "key": "k1", "label": "Label 1" },
+                          { "key": "k2", "label": "Label 2" } ]
+           A plain array of strings ("options": ["Label 1", "Label 2"]) also works.
+           Do NOT build a separate "select_options" block or set "option_layers" —
+           the system creates those from your inline options.
+
+        3) For RAW MEASUREMENT DATA, use type "upload" for a data-file field and type
+           "table" for a results table. Give a table its columns INLINE as:
+             "columns": [ { "name": "m/z", "type": "number" },
+                          { "name": "intensity", "type": "number" } ]
+           (a plain array of strings also works; column "type" is one of text,
+           number, integer, checkbox, date, datetime — default text). The system
+           builds the table structure from your columns.
+
+        Note: never set "option_layers", "value_system", "select_options" or
+        "sub_fields" yourself — the system derives them from your "unit" (numbers),
+        "options" (choices) and "columns" (tables).
+      PROMPT
+    end
+
+    def user_prompt
+      case @kind
+      when 'element', 'segment'
+        generic_user_prompt
+      else
+        dataset_user_prompt
+      end
+    end
+
+    def dataset_user_prompt
+      parts = []
+      parts << "Analytical method (CHMO ontology term): #{@ols_term_id}"
+      parts << "Template description / intent: #{@desc}" if @desc.present?
+      parts << "Default layout: set each layer's \"cols\" to #{@cols} (fields per row)."
+      if @references.any?
+        parts << 'Reference links (publications / standards such as DCAT, MICHI):'
+        @references.each { |ref| parts << "- #{ref}" }
+      end
+      file_context = files_context
+      parts << file_context if file_context.present?
+      parts << 'Design the dataset template now and return only the JSON object.'
+      parts.join("\n")
+    end
+
+    def generic_user_prompt
+      parts = []
+      parts << "#{@kind.capitalize} name/label: #{@subject}"
+      parts << "Template description / intent: #{@desc}" if @desc.present?
+      parts << "Default layout: set each layer's \"cols\" to #{@cols} (fields per row)."
+      if @references.any?
+        parts << 'Reference links (publications / standards):'
+        @references.each { |ref| parts << "- #{ref}" }
+      end
+      file_context = files_context
+      parts << file_context if file_context.present?
+      parts << "Design the #{@kind} template now and return only the JSON object."
+      parts.join("\n")
+    end
+
+    def files_context
+      usable = @files.first(MAX_FILES).map do |file|
+        name = file['filename'] || file[:filename]
+        next if name.blank?
+
+        # Prefer server-side extraction of the raw bytes (PDF / Excel / text);
+        # fall back to legacy client-read plaintext. FileExtractor never raises.
+        b64 = file['content_base64'] || file[:content_base64]
+        content = b64.present? ? Labimotion::FileExtractor.extract(name, b64) : (file['content'] || file[:content])
+        snippet = content.to_s[0, MAX_FILE_CHARS]
+        snippet.present? ? "### #{name}\n#{snippet}" : "### #{name} (no readable text)"
+      end.compact
+      return '' if usable.empty?
+
+      "Attached reference files (truncated):\n#{usable.join("\n\n")}"
+    end
+
+    # Build the chat messages for a refine request: a schema-aware system prompt,
+    # the current template as JSON, any prior conversation turns (for continuity)
+    # and the latest change request.
+    def refine_messages(current, instruction)
+      messages = [{ role: 'system', content: refine_system_content(current) }]
+      messages << { role: 'user', content: current_template_context(current) }
+      @history.each do |turn|
+        role = (turn['role'] || turn[:role]).to_s
+        content = (turn['content'] || turn[:content]).to_s
+        next if content.strip.blank?
+
+        messages << { role: (role == 'assistant' ? 'assistant' : 'user'), content: content }
+      end
+      messages << { role: 'user', content: refine_instruction_prompt(instruction) }
+      messages
+    end
+
+    def current_template_context(current)
+      current = {} unless current.is_a?(Hash)
+      layers = annotate_layer_groups(current['layers'] || current[:layers] || {}, current_metadata(current))
+      template = {
+        'label' => current['label'] || current[:label],
+        'layers' => layers,
+        'select_options' => current['select_options'] || current[:select_options] || {}
+      }
+      "Current dataset template (JSON):\n#{JSON.generate(template)}"
+    end
+
+    def current_metadata(current)
+      current['metadata'] || current[:metadata]
+    end
+
+    # Re-attach an existing metadata.groups block onto the layers as a "group" tag
+    # (the shape the model emits) so a refine round-trips grouping without the model
+    # having to reproduce the metadata block itself.
+    def annotate_layer_groups(layers, metadata)
+      return layers unless layers.is_a?(Hash)
+
+      groups = metadata_groups(metadata)
+      return layers if groups.empty?
+
+      apply_group_tags(layers, layer_group_map(groups))
+    end
+
+    def metadata_groups(metadata)
+      return [] unless metadata.is_a?(Hash)
+
+      groups = metadata['groups'] || metadata[:groups]
+      groups.is_a?(Array) ? groups : []
+    end
+
+    # { layer_key => group_label } from a metadata.groups array.
+    def layer_group_map(groups)
+      map = {}
+      groups.each do |grp|
+        next unless grp.is_a?(Hash)
+
+        label = (grp['label'] || grp[:label]).to_s
+        next if label.empty?
+
+        Array(grp['layers'] || grp[:layers]).each { |lk| map[lk.to_s] = label }
+      end
+      map
+    end
+
+    def apply_group_tags(layers, layer_group)
+      layers.each_with_object({}) do |(key, layer), out|
+        copy = layer.is_a?(Hash) ? layer.dup : layer
+        grp = layer_group[key.to_s]
+        copy['group'] = grp if grp.present? && copy.is_a?(Hash)
+        out[key] = copy
+      end
+    end
+
+    def refine_instruction_prompt(instruction)
+      <<~PROMPT.strip
+        Change request: #{instruction}
+
+        Apply this change to the current template. Preserve everything the request
+        does not mention (labels, fields, options, units and other properties).
+        Return the COMPLETE updated template as a single JSON object with keys
+        "label", "layers", "select_options" and a short "summary" (one sentence
+        describing what you changed). No prose, no markdown, no code fences.
+      PROMPT
+    end
+
+    def refine_system_prompt
+      cfg[:refine_system_prompt].presence || default_refine_system_prompt
+    end
+
+    def default_refine_system_prompt
+      <<~PROMPT
+        You are an assistant that edits metadata templates for analytical
+        chemistry datasets in the Chemotion ELN (LabIMotion generic dataset
+        framework).
+
+        You are given the current template as JSON and a change request. Apply the
+        requested change and return the COMPLETE updated template. Keep everything
+        the request does not mention exactly as it was, including any extra field
+        properties (units, sub-fields, restrictions, defaults).
+
+        Respond with a SINGLE JSON object only — no prose, no markdown, no code
+        fences — with exactly these keys: "label", "layers", "select_options" and
+        "summary" (one sentence describing what you changed). "layers" and
+        "select_options" keep the same shape as the input.
+
+        Rules (LabIMotion property schema):
+        - layer_key and field_key are lowercase snake_case ([a-z0-9_]), unique
+          within their scope.
+        - Each layer MUST include: key, label, cols (integer 1-6), color
+          ("default"), position (1-based integer) and fields (array).
+        - Each field MUST include: field, type, label, position, required.
+        - "type" MUST be one of: "text", "textarea", "integer", "number",
+          "checkbox", "date", "datetime", "select", "select-multi",
+          "ontology-select", "system-defined" (unless the input field already
+          uses another type, which you must keep).
+        - For a "select"/"select-multi" field, list the choices INLINE on the field
+          as "options": [ { "key": "k", "label": "Label" } ] (a plain string array
+          also works); do not set option_layers or a separate select_options block.
+        - Renumber "position" values consecutively within each layer and field
+          list after additions or removals.
+      PROMPT
+    end
+
+    # Refine normally edits an existing template. But when the current template
+    # is empty or has no usable fields, "editing" is really a design task — so
+    # swap in the generate/design system prompt (with a note to still return the
+    # refine "summary" key). This keeps a single user-chosen model and never asks
+    # the user to switch to "New (AI)": the code recognises the from-scratch case
+    # itself. A populated template keeps the edit prompt.
+    def refine_system_content(current)
+      parts =
+        if refine_from_scratch?(current)
+          [system_prompt, from_scratch_note, field_rules_prompt]
+        else
+          [refine_system_prompt, field_rules_prompt]
+        end
+      scope = method_scope_prompt
+      parts << scope if scope.present?
+      parts.join("\n\n")
+    end
+
+    # A dataset template is bound to a CHMO analytical-method term. Give the
+    # model that method as authoritative scope so it keeps the template coherent:
+    # add fields that fit the method or are general dataset metadata, and decline
+    # clearly-unrelated ones (explaining in the summary) instead of blindly adding
+    # whatever was asked. Blank when no term is known (e.g. the element/segment
+    # refine paths, which are not method-bound), so the guard is dataset-only.
+    def method_scope_prompt
+      id, label = method_term
+      return '' if id.blank? && label.blank?
+
+      method = label.present? ? "\"#{label}\" (#{id})" : id
+      <<~PROMPT.strip
+        This #{@kind} template records data for the analytical method #{method}.
+        Treat that method as the authoritative scope of the template. When
+        applying the change request, add or keep only fields that are relevant to
+        this method or are general dataset metadata (operator, date, sample id,
+        notes, funding and the like). If the request asks to add fields clearly
+        unrelated to this method — neither method-specific nor general metadata —
+        do NOT add them; keep the rest of the template valid and state in the
+        "summary" what you declined and why.
+      PROMPT
+    end
+
+    # Split a stored ols_term_id ("CHMO:0000470 | mass spectrometry") into its id
+    # and optional human label; a bare id returns a blank label.
+    def method_term
+      raw = @ols_term_id.to_s.strip
+      return ['', ''] if raw.blank?
+
+      id, label = raw.split('|', 2)
+      [id.to_s.strip, label.to_s.strip]
+    end
+
+    # True when there is nothing meaningful to edit: no layers, or no layer
+    # carries any real field. ("Nonsense but populated" is not code-detectable;
+    # only structural emptiness is.)
+    def refine_from_scratch?(current)
+      current = {} unless current.is_a?(Hash)
+      layers = current['layers'] || current[:layers] || {}
+      values = layers.respond_to?(:values) ? layers.values : Array(layers)
+      values.none? { |layer| layer_has_fields?(layer) }
+    end
+
+    def layer_has_fields?(layer)
+      return false unless layer.is_a?(Hash)
+
+      Array(layer['fields'] || layer[:fields]).any? do |f|
+        f.is_a?(Hash) && (f['field'] || f[:field]).present?
+      end
+    end
+
+    def from_scratch_note
+      <<~PROMPT.strip
+        The current template is empty or has no usable fields. Treat the change
+        request below as a request to DESIGN a complete template from scratch,
+        following the design guidance above. In addition to "label", "layers"
+        and "select_options", include a one-sentence "summary" key describing
+        what you built.
+      PROMPT
+    end
+
+    # Tolerate models that wrap JSON in ```json fences despite instructions.
+    def parse_json(text)
+      cleaned = text.strip
+      cleaned = cleaned.sub(/\A```(?:json)?\s*/i, '').sub(/\s*```\z/, '') if cleaned.start_with?('```')
+      JSON.parse(cleaned)
+    end
+
+    # Coerce the model output into a schema-valid shape, filling the keys the
+    # LabIMotion layers/field-base schemas require (layer: key/cols/color/
+    # position; field: field/type) so the template renders even if the model
+    # omits them.
+    def normalize(data)
+      data = {} unless data.is_a?(Hash)
+      raw_layers = data['layers'].is_a?(Hash) ? data['layers'] : {}
+      layers = {}
+      raw_layers.each_with_index do |(key, layer), idx|
+        built = normalize_layer(key.to_s, layer, idx)
+        carry_group_tag(built, layer) if layer.is_a?(Hash)
+        layers[key.to_s] = built
+      end
+      select_options = data['select_options'].is_a?(Hash) ? data['select_options'].dup : {}
+      lift_inline_options!(layers, select_options)
+      {
+        'label' => data['label'].to_s.strip,
+        'layers' => layers,
+        'select_options' => select_options,
+        'metadata' => metadata_for(data, layers)
+      }
+    end
+
+    def normalize_layer(key, layer, index)
+      layer = {} unless layer.is_a?(Hash)
+      raw_fields = layer['fields'].is_a?(Array) ? layer['fields'] : []
+      fields = raw_fields.each_with_index.map { |field, fidx| normalize_field(field, fidx) }.compact
+      {
+        'key' => (layer['key'].presence || key).to_s,
+        'label' => (layer['label'].presence || key).to_s,
+        'cols' => layer['cols'].is_a?(Integer) ? layer['cols'].clamp(1, 6) : @cols,
+        'color' => layer['color'].presence || 'default',
+        'position' => layer['position'].is_a?(Integer) ? layer['position'] : (index + 1),
+        'fields' => fields
+      }
+    end
+
+    # Copy a model layer's UI-section "group" tag onto the normalized layer (the
+    # generate path allowlists layer keys); build_metadata_groups then lifts it.
+    def carry_group_tag(result, layer)
+      group = layer['group']
+      result['group'] = group.to_s if group.is_a?(String) && group.present?
+    end
+
+    def normalize_field(field, position)
+      return nil unless field.is_a?(Hash)
+
+      name = field['field'].presence
+      return nil if name.blank?
+
+      result = {
+        'field' => name.to_s,
+        'type' => FIELD_TYPES.include?(field['type']) ? field['type'] : 'text',
+        'label' => (field['label'].presence || name).to_s,
+        'position' => field['position'].is_a?(Integer) ? field['position'] : (position + 1),
+        'required' => field['required'] == true
+      }
+      result['default'] = field['default'].to_s if field['default'].present?
+      # option_layers = unit-group key (system-defined) OR select_options key
+      # (select / select-multi). Keep a STRING one; value_system is the default unit
+      # for a system-defined field.
+      ol = field['option_layers']
+      result['option_layers'] = ol.to_s if ol.is_a?(String) && ol.present?
+      result['value_system'] = field['value_system'].to_s if field['value_system'].present?
+      # Carry an inline options list forward (or an array a model wrongly put in
+      # option_layers) so lift_inline_options! folds it into select_options.
+      inline = field['options']
+      inline = ol if inline.nil? && ol.is_a?(Array)
+      result['options'] = inline if inline.is_a?(Array)
+      apply_unit_mapping(result, field)
+      apply_table_columns(result, field)
+      result
+    end
+
+    # Refine variant of #normalize that PRESERVES any extra layer/field keys the
+    # model echoes back (units, sub_fields, restrictions, cond_fields, ...) so
+    # editing an existing rich template does not silently drop advanced
+    # properties. It only enforces the required keys on top of the model output.
+    def normalize_preserving(data)
+      data = {} unless data.is_a?(Hash)
+      raw_layers = data['layers'].is_a?(Hash) ? data['layers'] : {}
+      layers = {}
+      raw_layers.each_with_index do |(key, layer), idx|
+        layers[key.to_s] = normalize_layer_preserving(key.to_s, layer, idx)
+      end
+      select_options = data['select_options'].is_a?(Hash) ? data['select_options'].dup : {}
+      lift_inline_options!(layers, select_options)
+      {
+        'label' => data['label'].to_s.strip,
+        'layers' => layers,
+        'select_options' => select_options,
+        'metadata' => metadata_for(data, layers)
+      }
+    end
+
+    def normalize_layer_preserving(key, layer, index)
+      layer = {} unless layer.is_a?(Hash)
+      raw_fields = layer['fields'].is_a?(Array) ? layer['fields'] : []
+      fields = raw_fields.each_with_index.map { |field, fidx| normalize_field_preserving(field, fidx) }.compact
+      layer.merge(
+        'key' => (layer['key'].presence || key).to_s,
+        'label' => (layer['label'].presence || key).to_s,
+        'cols' => layer['cols'].is_a?(Integer) ? layer['cols'].clamp(1, 6) : @cols,
+        'color' => layer['color'].presence || 'default',
+        'position' => layer['position'].is_a?(Integer) ? layer['position'] : (index + 1),
+        'fields' => fields
+      )
+    end
+
+    def normalize_field_preserving(field, position)
+      return nil unless field.is_a?(Hash)
+
+      name = field['field'].presence
+      return nil if name.blank?
+
+      # Keep the model's type verbatim (an existing template may legitimately use
+      # a type outside FIELD_TYPES, e.g. "table"); only default it when blank.
+      result = field.merge(
+        'field' => name.to_s,
+        'type' => (field['type'].presence || 'text').to_s,
+        'label' => (field['label'].presence || name).to_s,
+        'position' => field['position'].is_a?(Integer) ? field['position'] : (position + 1),
+        'required' => field['required'] == true
+      )
+      apply_unit_mapping(result, field)
+      apply_table_columns(result, field)
+      result
+    end
+
+    # Fold each select/select-multi field's inline `options` into the shared
+    # select_options map: generate a stable key, register { options: [{key,label}] },
+    # point the field's option_layers at it, and drop the inline copy. Lets the model
+    # return choices inline (which weak models get right far more often) instead of
+    # maintaining the error-prone option_layers <-> select_options cross-reference.
+    def lift_inline_options!(layers, select_options)
+      layers.each_value do |layer|
+        next unless layer.is_a?(Hash)
+
+        Array(layer['fields']).each { |field| lift_field_options!(field, select_options) }
+      end
+    end
+
+    def lift_field_options!(field, select_options)
+      return unless field.is_a?(Hash)
+
+      inline = field.delete('options')
+      return unless SELECT_TYPES.include?(field['type'].to_s)
+      return if resolved_option_layers?(field, select_options)
+
+      pairs = normalize_option_pairs(inline || array_option_layers(field))
+      return if pairs.empty?
+
+      key = unique_option_key(field['field'].to_s, select_options)
+      select_options[key] = { 'options' => pairs }
+      field['option_layers'] = key
+    end
+
+    # True when the field already points at a real select_options entry (the model
+    # produced the full cross-reference itself) — leave that untouched.
+    def resolved_option_layers?(field, select_options)
+      ol = field['option_layers']
+      ol.is_a?(String) && ol.present? && select_options[ol].is_a?(Hash)
+    end
+
+    # A model that inlines choices may put the array in option_layers by mistake.
+    def array_option_layers(field)
+      ol = field['option_layers']
+      ol.is_a?(Array) ? ol : nil
+    end
+
+    # Coerce inline options — ["A","B"] or [{key,label}] / [{value,label}] — into
+    # [{'key'=>..,'label'=>..}] pairs; blanks dropped, keys de-duplicated.
+    def normalize_option_pairs(options)
+      return [] unless options.is_a?(Array)
+
+      seen = {}
+      options.each_with_object([]) do |opt, acc|
+        key, label = option_key_label(opt)
+        next if key.empty? || seen[key]
+
+        seen[key] = true
+        acc << { 'key' => key, 'label' => label.presence || key }
+      end
+    end
+
+    def option_key_label(opt)
+      if opt.is_a?(Hash)
+        key = (opt['key'] || opt['value'] || opt['label']).to_s.strip
+        [key, (opt['label'].presence || key).to_s.strip]
+      else
+        str = opt.to_s.strip
+        [str, str]
+      end
+    end
+
+    # A select_options key from the field name, numerically suffixed if the base is
+    # taken, so two fields never collide.
+    def unique_option_key(field_name, select_options)
+      base = "#{field_name}_options"
+      key = base
+      n = 2
+      while select_options.key?(key)
+        key = "#{base}_#{n}"
+        n += 1
+      end
+      key
+    end
+
+    # Turn a model-supplied physical unit into a system-defined unit group. The
+    # model marks a measured number with "unit": "<symbol>"; here we resolve it to
+    # option_layers (group) + value_system (the specific unit) so the field renders
+    # with the right unit picker. Unknown/ambiguous units fall back to plain
+    # "number"; an already-valid system-defined field is left untouched.
+    def apply_unit_mapping(result, raw)
+      result.delete('unit') if result.key?('unit')
+      return if unit_group_ready?(result)
+      return unless UNIT_CANDIDATE_TYPES.include?(result['type'].to_s)
+
+      unit = (raw['unit'] || raw[:unit]).to_s.strip
+      resolved = unit.empty? ? nil : resolve_unit(unit)
+      resolved ? assign_unit_group(result, resolved) : fallback_from_system_defined(result, unit)
+    end
+
+    # True when the field already declares a real unit group (the current shape a
+    # strong model or an edited template produces) — do not second-guess it.
+    def unit_group_ready?(result)
+      result['type'] == 'system-defined' && valid_unit_group?(result['option_layers'])
+    end
+
+    def assign_unit_group(result, resolved)
+      result['type'] = 'system-defined'
+      result['option_layers'] = resolved[:group]
+      result['value_system'] = resolved[:value_system]
+    end
+
+    # A "system-defined" field whose unit we could not map would render broken, so
+    # downgrade it to a plain number and log the raw unit for diagnosis.
+    def fallback_from_system_defined(result, unit)
+      return unless result['type'] == 'system-defined'
+
+      log_ai_response("unmapped unit '#{unit}' on field '#{result['field']}' -> number", unit)
+      result['type'] = 'number'
+      result.delete('option_layers')
+      result.delete('value_system')
+    end
+
+    def resolve_unit(token)
+      self.class.unit_index[self.class.normalize_unit_token(token)]
+    end
+
+    def valid_unit_group?(key)
+      key.is_a?(String) && self.class.unit_groups.include?(key)
+    end
+
+    # Build a "table" field's sub_fields (columns) from the model's inline `columns`
+    # (or normalize an echoed sub_fields array on refine). Each column becomes
+    # { id, type, value:"", col_name } — the shape chem-generic-ui renders. Also sets
+    # text_sub_fields:[]. Non-table fields are untouched.
+    def apply_table_columns(result, raw)
+      return unless result['type'].to_s == 'table'
+
+      result['sub_fields'] = table_sub_fields(raw)
+      result['text_sub_fields'] = [] unless result['text_sub_fields'].is_a?(Array)
+      result.delete('columns')
+    end
+
+    def table_sub_fields(raw)
+      cols = raw['columns'] || raw[:columns]
+      return build_table_columns(cols) if cols.is_a?(Array)
+
+      existing = raw['sub_fields'] || raw[:sub_fields]
+      existing.is_a?(Array) ? keep_table_columns(existing) : []
+    end
+
+    def build_table_columns(cols)
+      seen = {}
+      cols.filter_map do |col|
+        name, type = table_col_name_type(col)
+        next if name.empty? || seen[name]
+
+        seen[name] = true
+        { 'id' => SecureRandom.uuid, 'type' => type, 'value' => '', 'col_name' => name }
+      end
+    end
+
+    def table_col_name_type(col)
+      return [col.to_s.strip, 'text'] unless col.is_a?(Hash)
+
+      name = (col['name'] || col['col_name'] || col['label'] || col['field']).to_s.strip
+      type = col['type'].to_s
+      [name, TABLE_COL_TYPES.include?(type) ? type : 'text']
+    end
+
+    # Keep an existing table's columns (refine echo), enforcing id/type/value/col_name
+    # while preserving any extra column keys (option_layers, value_system, ...).
+    def keep_table_columns(sub_fields)
+      sub_fields.filter_map do |sf|
+        next unless sf.is_a?(Hash)
+
+        name = (sf['col_name'] || sf[:col_name]).to_s.strip
+        next if name.empty?
+
+        sf.merge('id' => (sf['id'].presence || SecureRandom.uuid).to_s,
+                 'type' => (sf['type'].presence || 'text').to_s,
+                 'value' => sf['value'].to_s, 'col_name' => name)
+      end
+    end
+
+    # The metadata.groups block (UI sectioning). Built from the per-layer "group"
+    # tag the model attaches; falls back to an existing metadata block (refine, when
+    # no layer is tagged). {} when neither is present.
+    def metadata_for(data, layers)
+      grouped = build_metadata_groups(layers)
+      return grouped unless grouped.empty?
+
+      existing = data['metadata'] || data[:metadata]
+      existing.is_a?(Hash) ? existing : {}
+    end
+
+    # Collect layers by their "group" tag (first-seen order = section order) into a
+    # schema-valid metadata.groups array, and strip the tag off each layer. Returns
+    # {} when no layer is grouped.
+    def build_metadata_groups(layers)
+      members = collect_layer_groups(layers)
+      return {} if members.empty?
+
+      groups = members.keys.each_with_index.map do |label, idx|
+        { 'id' => group_id(label, idx), 'label' => label, 'position' => idx + 1, 'layers' => members[label] }
+      end
+      { 'groups' => groups, 'restrict' => nil }
+    end
+
+    # layer "group" tag -> ordered { label => [layer_keys] }, stripping the tag.
+    # Ruby preserves insertion order, so the first-seen group is the first section.
+    def collect_layer_groups(layers)
+      members = {}
+      layers.each_value do |layer|
+        label = (layer.is_a?(Hash) ? layer.delete('group') : nil).to_s.strip
+        next if label.empty?
+
+        (members[label] ||= []) << layer['key'].to_s
+      end
+      members
+    end
+
+    # A slug id for a metadata group label; index keeps it non-empty and stable.
+    def group_id(label, idx)
+      slug = label.downcase.gsub(/[^a-z0-9]+/, '_').gsub(/\A_+|_+\z/, '')
+      slug.empty? ? "group_#{idx + 1}" : slug
+    end
+
+    # ---- data-fill (value extraction) -------------------------------------
+
+    # Chat messages for a data-fill request: a value-extraction system prompt,
+    # then the compact field schema + the document text in the user turn.
+    def fill_messages(schema, context_text)
+      [
+        { role: 'system', content: fill_system_prompt },
+        { role: 'user', content: fill_user_prompt(schema, context_text) }
+      ]
+    end
+
+    def fill_system_prompt
+      cfg[:fill_system_prompt].presence || default_fill_system_prompt
+    end
+
+    def default_fill_system_prompt
+      <<~PROMPT
+        You are an assistant that extracts structured metadata VALUES from a
+        scientific document to pre-fill a LabIMotion generic element's form fields.
+
+        You are given (1) a compact schema of the element's fields, grouped by
+        layer, and (2) the text extracted from a document. Read the document and
+        return the values you can find for those fields.
+
+        Respond with a SINGLE JSON object only — no prose, no markdown, no code
+        fences — of exactly this shape:
+
+        { "values": { "<layer_key>": { "<field_key>": <value> } },
+          "summary": "<one short sentence on what you found>" }
+
+        Rules:
+        - Only include a field whose value you ACTUALLY FIND in the document. OMIT
+          every other field. Never invent, guess or infer a value the text does not
+          support. When unsure, leave the field out.
+        - Use the exact layer_key and field_key from the schema.
+        - "integer" / "number" / "system-defined" -> a JSON number only (no unit
+          text; the unit is fixed by the template).
+        - "checkbox" -> true or false.
+        - "date" -> "YYYY-MM-DD". "datetime" -> ISO-8601.
+        - "select" / "select-multi": each such field lists its allowed options as
+          {"key","label"} pairs. Match the document wording against an option's
+          "label" (or "key") and return the option's "key" — a single key for
+          "select", an array of keys for "select-multi". Only use keys from that
+          field's own options; omit the field if none apply.
+        - "text" / "textarea" -> a string.
+        - If you find no values at all, return { "values": {}, "summary": "..." }.
+      PROMPT
+    end
+
+    def fill_user_prompt(schema, context_text)
+      <<~PROMPT
+        Element field schema (JSON):
+        #{JSON.generate(schema)}
+
+        Document text:
+        #{context_text}
+
+        Extract the field values now and return only the JSON object.
+      PROMPT
+    end
+
+    # A COMPACT schema of the template's fillable fields, grouped by layer:
+    # [{ 'layer' => k, 'label' => l, 'fields' => [{ 'field', 'label', 'type',
+    # ('options' => [{ 'key', 'label' }] for select/select-multi) }] }]. Layers
+    # with no fields are dropped so the model isn't shown empty groups.
+    def fill_field_schema(properties)
+      layers = properties['layers'].is_a?(Hash) ? properties['layers'] : {}
+      select_options = properties['select_options'].is_a?(Hash) ? properties['select_options'] : {}
+      schema = []
+      layers.each do |layer_key, layer|
+        next unless layer.is_a?(Hash)
+
+        raw_fields = layer['fields'].is_a?(Array) ? layer['fields'] : []
+        fields = raw_fields.map { |field| fill_schema_field(field, select_options) }.compact
+        next if fields.empty?
+
+        schema << {
+          'layer' => layer_key.to_s,
+          'label' => (layer['label'].presence || layer_key).to_s,
+          'fields' => fields
+        }
+      end
+      schema
+    end
+
+    def fill_schema_field(field, select_options)
+      return nil unless field.is_a?(Hash)
+
+      name = field['field'].presence
+      return nil if name.blank?
+
+      type = field['type'].to_s
+      entry = { 'field' => name.to_s, 'label' => (field['label'].presence || name).to_s, 'type' => type }
+      entry['options'] = fill_option_pairs(select_options, field) if %w[select select-multi].include?(type)
+      entry
+    end
+
+    # Allowed options for a select/select-multi field as { 'key' => .., 'label' => .. }
+    # pairs, read from properties['select_options'][field['option_layers']]['options'].
+    # Labels are sent to the model so it can map the document wording (which is
+    # phrased like the human-readable label, not the machine key) to an option.
+    def fill_option_pairs(select_options, field)
+      group = select_options[field['option_layers']]
+      options = group.is_a?(Hash) ? group['options'] : nil
+      return [] unless options.is_a?(Array)
+
+      options.filter_map do |opt|
+        next unless opt.is_a?(Hash)
+
+        opt_key = opt['key'].presence
+        next if opt_key.nil?
+
+        { 'key' => opt_key.to_s, 'label' => (opt['label'].presence || opt_key).to_s }
+      end
+    end
+
+    # Map a model answer — which may be an option KEY or the human LABEL — back to
+    # the option key. Exact key wins, then a case-insensitive label-or-key match;
+    # nil when nothing matches (the value is dropped upstream).
+    def resolve_option_key(pairs, candidate)
+      cand = candidate.to_s.strip
+      return nil if cand.empty?
+
+      exact = pairs.find { |p| p['key'] == cand }
+      return exact['key'] if exact
+
+      ci = cand.downcase
+      match = pairs.find { |p| p['label'].downcase == ci || p['key'].downcase == ci }
+      match && match['key']
+    end
+
+    # Validate the model's { layer => { field => value } } against the template:
+    # keep only existing layer/field keys, coerce per field type, drop select
+    # values outside the allowed option keys, and drop nil/blank values.
+    def normalize_fill_values(properties, raw_values)
+      return {} unless raw_values.is_a?(Hash)
+
+      layers = properties['layers'].is_a?(Hash) ? properties['layers'] : {}
+      select_options = properties['select_options'].is_a?(Hash) ? properties['select_options'] : {}
+      result = {}
+      raw_values.each do |layer_key, fields|
+        layer = layers[layer_key.to_s]
+        next unless layer.is_a?(Hash) && fields.is_a?(Hash)
+
+        field_defs = fill_field_defs(layer)
+        clean = {}
+        fields.each do |field_key, value|
+          field = field_defs[field_key.to_s]
+          next if field.nil?
+
+          coerced = coerce_fill_value(field, value, select_options)
+          next if fill_blank?(coerced)
+
+          clean[field_key.to_s] = coerced
+        end
+        result[layer_key.to_s] = clean unless clean.empty?
+      end
+      result
+    end
+
+    def fill_field_defs(layer)
+      raw_fields = layer['fields'].is_a?(Array) ? layer['fields'] : []
+      raw_fields.each_with_object({}) do |field, memo|
+        next unless field.is_a?(Hash)
+
+        name = field['field'].presence
+        memo[name.to_s] = field if name.present?
+      end
+    end
+
+    def fill_blank?(value)
+      return true if value.nil?
+      return true if value.is_a?(String) && value.strip.empty?
+      return true if value.is_a?(Array) && value.empty?
+
+      false
+    end
+
+    def coerce_fill_value(field, value, select_options)
+      return nil if value.nil?
+
+      case field['type'].to_s
+      when 'integer'
+        num = fill_to_number(value)
+        num.nil? ? nil : num.to_i
+      when 'number', 'system-defined'
+        fill_to_number(value)
+      when 'checkbox'
+        fill_to_boolean(value)
+      when 'select'
+        pairs = fill_option_pairs(select_options, field)
+        candidate = value.is_a?(Array) ? value.first : value
+        resolve_option_key(pairs, candidate)
+      when 'select-multi'
+        pairs = fill_option_pairs(select_options, field)
+        list = value.is_a?(Array) ? value : [value]
+        list.filter_map { |v| resolve_option_key(pairs, v) }.uniq
+      else
+        # text / textarea / date / datetime / ontology-select / label / other:
+        # keep scalars as a string; drop objects/arrays we can't render.
+        return value.to_s if value.is_a?(String) || value.is_a?(Numeric)
+        return value.to_s if value == true || value == false
+
+        nil
+      end
+    end
+
+    def fill_to_number(value)
+      case value
+      when Numeric
+        value
+      when String
+        str = value.strip
+        return nil if str.empty?
+        return str.to_i if str.match?(/\A[+-]?\d+\z/)
+        return str.to_f if str.match?(/\A[+-]?\d*\.?\d+(?:[eE][+-]?\d+)?\z/)
+
+        nil
+      end
+    end
+
+    def fill_to_boolean(value)
+      case value
+      when true, false
+        value
+      when Numeric
+        !value.zero?
+      when String
+        case value.strip.downcase
+        when 'true', 'yes', '1' then true
+        when 'false', 'no', '0' then false
+        end
+      end
+    end
+  end
+end