legion-data 1.6.4 → 1.6.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4affe9deaa1903b9ff010b805e8f37814bec22ca669186daf69de345a5e8d711
-  data.tar.gz: 6e78bf1992c58cd92f9c05395486f1b1c6f211ae3ad653fc385240563bef96ba
+  metadata.gz: 55c32ec7725b3d773d0050f52873892de6d9091082546c25969288a7abe159a9
+  data.tar.gz: 9d837ec4d10952b7103373972354d32886737f845bd623ad09d3671eddcd9f1d
 SHA512:
-  metadata.gz: 8a16917741977b4bd9c3ed8b873769dc6f3356cdb950a2bef2abd9809cbc340ec2df966bc71700e5a6b116d86453988f3cbd79db5b9f828e74dabba350e106a8
-  data.tar.gz: 9f32dc1dd38e9a9181b0f799509c06e1dd58e0661251426a135cf04bad8e64ca39f38f8ec93adc11e5715bbb3d0c95e878d4c82ebd9977425ace69be3b23cd84
+  metadata.gz: c4d314992432f8b4e6c67b1d8c69bc60d0f0da5c28f0e7906ecd455e8d12e8c7756d32b99fa96e7ce57ed2c1478ce88f1ffe8a0d17f9fdd0fa0896e5501d0ac4
+  data.tar.gz: b98331ae6bb79d68525cbe361ec054d0919ee14ee1e4322c3d191ad0d15f86c1c587e33ff358cf5fdd824b37e47ae40ad7b7d73d96dc6cbf38b917d6f01c7a5c

data/CHANGELOG.md

@@ -1,5 +1,20 @@
 # Legion::Data Changelog
 
+## [1.6.6] - 2026-03-25
+
+### Added
+- `connected?` — returns true when the shared DB is connected (reads `Settings[:data][:connected]`)
+- `can_write?(table_name)` — checks INSERT privilege; sqlite always returns true, postgres queries `has_table_privilege`, results cached per table
+- `can_read?(table_name)` — checks SELECT privilege; sqlite always returns true, postgres queries `has_table_privilege`, results cached per table
+- `reset_privileges!` — clears cached privilege results (used in tests and after re-connect)
+- `Legion::Data::Extract` — file format extraction with handler registry
+- Built-in handlers: text, markdown, csv, json, jsonl (no external gems required)
+- Optional handlers: pdf (pdf-reader), docx (docx), pptx (rubyzip), xlsx (rubyXL), html (nokogiri) — lazy-loaded, degrade gracefully if gem not installed
+- `Extract.register_handler(type, klass)` — register custom format handlers
+- `Extract.can_extract?(type)` — check if a type can be extracted (handler present and gem available)
+- `Extract.supported_types` — list all registered types
+- Added `csv` gem dependency (Ruby 3.4 stdlib split)
+
 ## [1.6.4] - 2026-03-25
 
 ### Added

data/legion-data.gemspec

@@ -26,6 +26,7 @@ Gem::Specification.new do |spec|
     'rubygems_mfa_required' => 'true'
   }
 
+  spec.add_dependency 'csv', '>= 3.2'
   spec.add_dependency 'legion-logging', '>= 1.2.8'
   spec.add_dependency 'legion-settings', '>= 1.3.12'
   spec.add_dependency 'sequel', '>= 5.70'

data/lib/legion/data/extract/handlers/base.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Legion
+  module Data
+    module Extract
+      module Handlers
+        class Base
+          @registry = {}
+
+          class << self
+            attr_reader :registry
+
+            def inherited(subclass)
+              super
+              # Deferred registration — subclass defines type after class body loads
+              TracePoint.new(:end) do |tp|
+                if tp.self == subclass
+                  register(subclass) if subclass.respond_to?(:type) && subclass.type
+                  tp.disable
+                end
+              end.enable
+            end
+
+            def register(handler_class)
+              @registry[handler_class.type] = handler_class
+            end
+
+            def for_type(type)
+              @registry[type&.to_sym]
+            end
+
+            def supported_types
+              @registry.keys
+            end
+
+            # Override in subclasses
+            def type = nil
+            def extensions = []
+            def gem_name = nil
+
+            def extract(_source)
+              raise NotImplementedError, "#{name} must implement .extract"
+            end
+
+            def available?
+              return true if gem_name.nil?
+
+              require gem_name
+              true
+            rescue LoadError
+              false
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/data/extract/handlers/csv.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require 'csv'
+
+module Legion
+  module Data
+    module Extract
+      module Handlers
+        class Csv < Base
+          def self.type = :csv
+          def self.extensions = %w[.csv]
+          def self.gem_name = nil
+
+          def self.extract(source)
+            content = source.respond_to?(:read) ? source.read : File.read(source.to_s)
+            table = ::CSV.parse(content, headers: true)
+            text = table.map { |row| row.to_h.map { |k, v| "#{k}: #{v}" }.join(', ') }.join("\n")
+            { text: text, metadata: { rows: table.size, columns: table.headers.size, headers: table.headers } }
+          rescue StandardError => e
+            { text: nil, error: e.message }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/data/extract/handlers/docx.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Legion
+  module Data
+    module Extract
+      module Handlers
+        class Docx < Base
+          def self.type = :docx
+          def self.extensions = %w[.docx]
+          def self.gem_name = 'docx'
+
+          def self.extract(source)
+            require 'docx'
+
+            doc = ::Docx::Document.open(source)
+            paragraphs = doc.paragraphs.map(&:text).reject(&:empty?)
+            text = paragraphs.join("\n\n")
+            { text: text, metadata: { paragraphs: paragraphs.size } }
+          rescue LoadError
+            { text: nil, error: :gem_not_installed, gem: gem_name }
+          rescue StandardError => e
+            { text: nil, error: e.message }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/data/extract/handlers/html.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Legion
+  module Data
+    module Extract
+      module Handlers
+        class Html < Base
+          def self.type = :html
+          def self.extensions = %w[.html .htm]
+          def self.gem_name = 'nokogiri'
+
+          def self.extract(source)
+            require 'nokogiri'
+
+            content = source.respond_to?(:read) ? source.read : File.read(source.to_s)
+            doc = ::Nokogiri::HTML(content)
+
+            # Remove script and style elements
+            doc.css('script, style, noscript').each(&:remove)
+
+            title = doc.at_css('title')&.text&.strip
+            text = doc.text.gsub(/\s+/, ' ').strip
+            { text: text, metadata: { title: title } }
+          rescue LoadError
+            { text: nil, error: :gem_not_installed, gem: gem_name }
+          rescue StandardError => e
+            { text: nil, error: e.message }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/data/extract/handlers/json.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module Legion
+  module Data
+    module Extract
+      module Handlers
+        class Json < Base
+          def self.type = :json
+          def self.extensions = %w[.json]
+          def self.gem_name = nil
+
+          def self.extract(source)
+            content = source.respond_to?(:read) ? source.read : File.read(source.to_s)
+            parsed = ::JSON.parse(content)
+            text = ::JSON.pretty_generate(parsed)
+            { text: text, metadata: { keys: parsed.is_a?(Hash) ? parsed.keys : nil } }
+          rescue StandardError => e
+            { text: nil, error: e.message }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/data/extract/handlers/jsonl.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module Legion
+  module Data
+    module Extract
+      module Handlers
+        class Jsonl < Base
+          def self.type = :jsonl
+          def self.extensions = %w[.jsonl]
+          def self.gem_name = nil
+
+          def self.extract(source)
+            content = source.respond_to?(:read) ? source.read : File.read(source.to_s)
+            lines = content.each_line.map { |l| ::JSON.parse(l.strip) rescue l.strip } # rubocop:disable Style/RescueModifier
+            text = lines.map { |l| l.is_a?(Hash) ? ::JSON.pretty_generate(l) : l }.join("\n---\n")
+            { text: text, metadata: { lines: lines.size } }
+          rescue StandardError => e
+            { text: nil, error: e.message }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/data/extract/handlers/markdown.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Legion
+  module Data
+    module Extract
+      module Handlers
+        class Markdown < Base
+          def self.type = :markdown
+          def self.extensions = %w[.md .markdown]
+          def self.gem_name = nil
+
+          def self.extract(source)
+            content = source.respond_to?(:read) ? source.read : File.read(source.to_s)
+            # Strip YAML frontmatter if present
+            text = content.sub(/\A---\n.*?\n---\n/m, '')
+            { text: text.strip, metadata: { bytes: content.bytesize, has_frontmatter: content != text } }
+          rescue StandardError => e
+            { text: nil, error: e.message }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/data/extract/handlers/pdf.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Legion
+  module Data
+    module Extract
+      module Handlers
+        class Pdf < Base
+          def self.type = :pdf
+          def self.extensions = %w[.pdf]
+          def self.gem_name = 'pdf-reader'
+
+          def self.extract(source)
+            require 'pdf-reader'
+
+            reader = ::PDF::Reader.new(source)
+            text = reader.pages.map(&:text).join("\n\n")
+            { text: text, metadata: { pages: reader.page_count, title: reader.info[:Title] } }
+          rescue LoadError
+            { text: nil, error: :gem_not_installed, gem: gem_name }
+          rescue StandardError => e
+            { text: nil, error: e.message }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/data/extract/handlers/pptx.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Legion
+  module Data
+    module Extract
+      module Handlers
+        class Pptx < Base
+          def self.type = :pptx
+          def self.extensions = %w[.pptx]
+          def self.gem_name = 'rubyzip'
+
+          def self.extract(source)
+            require 'zip'
+            require 'rexml/document'
+
+            slides = []
+            ::Zip::File.open(source) do |zip|
+              zip.glob('ppt/slides/slide*.xml').sort_by(&:name).each do |entry|
+                doc = REXML::Document.new(entry.get_input_stream.read)
+                texts = []
+                doc.each_element('//a:t') { |e| texts << e.text }
+                slides << texts.join(' ') unless texts.empty?
+              end
+            end
+            text = slides.each_with_index.map { |s, i| "Slide #{i + 1}: #{s}" }.join("\n\n")
+            { text: text, metadata: { slides: slides.size } }
+          rescue LoadError
+            { text: nil, error: :gem_not_installed, gem: 'rubyzip' }
+          rescue StandardError => e
+            { text: nil, error: e.message }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/data/extract/handlers/text.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Legion
+  module Data
+    module Extract
+      module Handlers
+        class Text < Base
+          def self.type = :text
+          def self.extensions = %w[.txt]
+          def self.gem_name = nil
+
+          def self.extract(source)
+            content = source.respond_to?(:read) ? source.read : File.read(source.to_s)
+            { text: content, metadata: { bytes: content.bytesize } }
+          rescue StandardError => e
+            { text: nil, error: e.message }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/data/extract/handlers/xlsx.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Legion
+  module Data
+    module Extract
+      module Handlers
+        class Xlsx < Base
+          def self.type = :xlsx
+          def self.extensions = %w[.xlsx .xls]
+          def self.gem_name = 'rubyXL'
+
+          def self.extract(source)
+            require 'rubyXL'
+            require 'rubyXL/convenience_methods'
+
+            workbook = ::RubyXL::Parser.parse(source)
+            sheets = []
+            workbook.worksheets.each do |sheet|
+              rows = sheet.each.map do |row|
+                next unless row
+
+                row.cells.map { |c| c&.value.to_s }.join(', ')
+              end.compact
+              sheets << "Sheet: #{sheet.sheet_name}\n#{rows.join("\n")}" unless rows.empty?
+            end
+            text = sheets.join("\n\n")
+            { text: text, metadata: { sheets: workbook.worksheets.size } }
+          rescue LoadError
+            { text: nil, error: :gem_not_installed, gem: gem_name }
+          rescue StandardError => e
+            { text: nil, error: e.message }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/data/extract/type_detector.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Legion
+  module Data
+    module Extract
+      module TypeDetector
+        EXTENSION_MAP = {
+          '.pdf'      => :pdf,
+          '.docx'     => :docx,
+          '.pptx'     => :pptx,
+          '.xlsx'     => :xlsx,
+          '.xls'      => :xlsx,
+          '.md'       => :markdown,
+          '.markdown' => :markdown,
+          '.txt'      => :text,
+          '.csv'      => :csv,
+          '.json'     => :json,
+          '.jsonl'    => :jsonl,
+          '.html'     => :html,
+          '.htm'      => :html
+        }.freeze
+
+        module_function
+
+        def detect(source)
+          return detect_from_path(source) if source.is_a?(String) && File.exist?(source)
+          return detect_from_io(source) if source.respond_to?(:path)
+
+          nil
+        end
+
+        def detect_from_path(path)
+          ext = File.extname(path).downcase
+          EXTENSION_MAP[ext]
+        end
+
+        def detect_from_io(io)
+          return nil unless io.respond_to?(:path) && io.path
+
+          detect_from_path(io.path)
+        end
+      end
+    end
+  end
+end

data/lib/legion/data/extract.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require_relative 'extract/type_detector'
+require_relative 'extract/handlers/base'
+
+module Legion
+  module Data
+    module Extract
+      class << self
+        def extract(source, type: :auto)
+          detected_type = type == :auto ? TypeDetector.detect(source) : type&.to_sym
+          return { success: false, text: nil, error: :unknown_type } unless detected_type
+
+          handler = Handlers::Base.for_type(detected_type)
+          return { success: false, text: nil, error: :no_handler, type: detected_type } unless handler
+
+          unless handler.available?
+            return { success: false, text: nil, error: :gem_not_installed,
+                     gem: handler.gem_name, type: detected_type }
+          end
+
+          result = handler.extract(source)
+          if result[:text]
+            { success: true, text: result[:text], metadata: result[:metadata], type: detected_type }
+          else
+            { success: false, text: nil, error: result[:error], type: detected_type }
+          end
+        rescue StandardError => e
+          { success: false, text: nil, error: e.message, type: detected_type }
+        end
+
+        def supported_types
+          load_all_handlers
+          Handlers::Base.supported_types
+        end
+
+        def can_extract?(type)
+          load_all_handlers
+          handler = Handlers::Base.for_type(type&.to_sym)
+          handler&.available? || false
+        end
+
+        def register_handler(type, klass)
+          Handlers::Base.registry[type.to_sym] = klass
+        end
+
+        private
+
+        def load_all_handlers
+          return if @handlers_loaded
+
+          Dir[File.join(__dir__, 'extract', 'handlers', '*.rb')].each do |f|
+            require f unless f.end_with?('base.rb')
+          end
+          @handlers_loaded = true
+        end
+      end
+    end
+  end
+end

data/lib/legion/data/version.rb

@@ -2,6 +2,6 @@
 
 module Legion
   module Data
-    VERSION = '1.6.4'
+    VERSION = '1.6.6'
   end
 end

data/lib/legion/data.rb

@@ -13,6 +13,7 @@ require_relative 'data/partition_manager'
 require_relative 'data/archiver'
 require_relative 'data/helper'
 require_relative 'data/rls'
+require_relative 'data/extract'
 
 module Legion
   module Data
@@ -55,6 +56,51 @@ module Legion
         }
       end
 
+      def connected?
+        Legion::Settings[:data][:connected] == true
+      rescue StandardError
+        false
+      end
+
+      def can_write?(table_name)
+        return false unless connected?
+
+        adapter = Legion::Settings[:data][:adapter]&.to_s
+        return true if adapter == 'sqlite'
+
+        @write_privileges ||= {}
+        return @write_privileges[table_name] unless @write_privileges[table_name].nil?
+
+        @write_privileges[table_name] = connection
+                                        .fetch("SELECT has_table_privilege(current_user, ?, 'INSERT') AS can", table_name.to_s)
+                                        .first[:can] == true
+      rescue StandardError
+        @write_privileges[table_name] = false if @write_privileges
+        false
+      end
+
+      def can_read?(table_name)
+        return false unless connected?
+
+        adapter = Legion::Settings[:data][:adapter]&.to_s
+        return true if adapter == 'sqlite'
+
+        @read_privileges ||= {}
+        return @read_privileges[table_name] unless @read_privileges[table_name].nil?
+
+        @read_privileges[table_name] = connection
+                                       .fetch("SELECT has_table_privilege(current_user, ?, 'SELECT') AS can", table_name.to_s)
+                                       .first[:can] == true
+      rescue StandardError
+        @read_privileges[table_name] = false if @read_privileges
+        false
+      end
+
+      def reset_privileges!
+        @write_privileges = nil
+        @read_privileges = nil
+      end
+
       def setup_cache
         cache_settings = Legion::Settings[:data][:cache]
         setup_static_cache if cache_settings[:static_cache]

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: legion-data
 version: !ruby/object:Gem::Version
-  version: 1.6.4
+  version: 1.6.6
 platform: ruby
 authors:
 - Esity
@@ -9,6 +9,20 @@ bindir: bin
 cert_chain: []
 date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: csv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.2'
 - !ruby/object:Gem::Dependency
   name: legion-logging
   requirement: !ruby/object:Gem::Requirement
@@ -98,6 +112,19 @@ files:
 - lib/legion/data/encryption/sequel_plugin.rb
 - lib/legion/data/event_store.rb
 - lib/legion/data/event_store/projection.rb
+- lib/legion/data/extract.rb
+- lib/legion/data/extract/handlers/base.rb
+- lib/legion/data/extract/handlers/csv.rb
+- lib/legion/data/extract/handlers/docx.rb
+- lib/legion/data/extract/handlers/html.rb
+- lib/legion/data/extract/handlers/json.rb
+- lib/legion/data/extract/handlers/jsonl.rb
+- lib/legion/data/extract/handlers/markdown.rb
+- lib/legion/data/extract/handlers/pdf.rb
+- lib/legion/data/extract/handlers/pptx.rb
+- lib/legion/data/extract/handlers/text.rb
+- lib/legion/data/extract/handlers/xlsx.rb
+- lib/legion/data/extract/type_detector.rb
 - lib/legion/data/helper.rb
 - lib/legion/data/local.rb
 - lib/legion/data/migration.rb