legion-data 1.6.3 → 1.6.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '048cfb8d6ead50ef6ddc554765251d61121dc0f6e99491b9675e410f086cd2fe'
-  data.tar.gz: 58cced043937061ce27fd3c111d5b0b5c5fd3416ab622a3a7700f1ce553628a4
+  metadata.gz: 55c32ec7725b3d773d0050f52873892de6d9091082546c25969288a7abe159a9
+  data.tar.gz: 9d837ec4d10952b7103373972354d32886737f845bd623ad09d3671eddcd9f1d
 SHA512:
-  metadata.gz: 9ba6d97c8cee58a5f41cd5ecc5c2229e75d4880368c43ec44d2f3b969ff144c81c7d6f49fd91d84819b1d3665eab37a7f7ff08f22c0e579109a96ace83d24262
-  data.tar.gz: ba69290675fbf53d9185dcdbc62e46e344b6aad3594c7dc4fd76fdf48af386a2b97d5b1e7fe70606b1c91dacee32e5a4117d6cc5ccf34a184fac89f4ce7b2e26
+  metadata.gz: c4d314992432f8b4e6c67b1d8c69bc60d0f0da5c28f0e7906ecd455e8d12e8c7756d32b99fa96e7ce57ed2c1478ce88f1ffe8a0d17f9fdd0fa0896e5501d0ac4
+  data.tar.gz: b98331ae6bb79d68525cbe361ec054d0919ee14ee1e4322c3d191ad0d15f86c1c587e33ff358cf5fdd824b37e47ae40ad7b7d73d96dc6cbf38b917d6f01c7a5c

data/CHANGELOG.md

@@ -1,5 +1,25 @@
 # Legion::Data Changelog
 
+## [1.6.6] - 2026-03-25
+
+### Added
+- `connected?` — returns true when the shared DB is connected (reads `Settings[:data][:connected]`)
+- `can_write?(table_name)` — checks INSERT privilege; sqlite always returns true, postgres queries `has_table_privilege`, results cached per table
+- `can_read?(table_name)` — checks SELECT privilege; sqlite always returns true, postgres queries `has_table_privilege`, results cached per table
+- `reset_privileges!` — clears cached privilege results (used in tests and after re-connect)
+- `Legion::Data::Extract` — file format extraction with handler registry
+- Built-in handlers: text, markdown, csv, json, jsonl (no external gems required)
+- Optional handlers: pdf (pdf-reader), docx (docx), pptx (rubyzip), xlsx (rubyXL), html (nokogiri) — lazy-loaded, degrade gracefully if gem not installed
+- `Extract.register_handler(type, klass)` — register custom format handlers
+- `Extract.can_extract?(type)` — check if a type can be extracted (handler present and gem available)
+- `Extract.supported_types` — list all registered types
+- Added `csv` gem dependency (Ruby 3.4 stdlib split)
+
+## [1.6.4] - 2026-03-25
+
+### Added
+- Migration 047: Apollo identity columns (submitted_by, submitted_from), content hash dedup, apollo_operations table, apollo_entries_archive table, comprehensive indexes including partial HNSW on active entries only
+
 ## [1.6.2] - 2026-03-25
 
 ### Changed

data/legion-data.gemspec

@@ -26,6 +26,7 @@ Gem::Specification.new do |spec|
     'rubygems_mfa_required' => 'true'
   }
 
+  spec.add_dependency 'csv', '>= 3.2'
   spec.add_dependency 'legion-logging', '>= 1.2.8'
   spec.add_dependency 'legion-settings', '>= 1.3.12'
   spec.add_dependency 'sequel', '>= 5.70'

data/lib/legion/data/extract/handlers/base.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Legion
+  module Data
+    module Extract
+      module Handlers
+        class Base
+          @registry = {}
+
+          class << self
+            attr_reader :registry
+
+            def inherited(subclass)
+              super
+              # Deferred registration — subclass defines type after class body loads
+              TracePoint.new(:end) do |tp|
+                if tp.self == subclass
+                  register(subclass) if subclass.respond_to?(:type) && subclass.type
+                  tp.disable
+                end
+              end.enable
+            end
+
+            def register(handler_class)
+              @registry[handler_class.type] = handler_class
+            end
+
+            def for_type(type)
+              @registry[type&.to_sym]
+            end
+
+            def supported_types
+              @registry.keys
+            end
+
+            # Override in subclasses
+            def type = nil
+            def extensions = []
+            def gem_name = nil
+
+            def extract(_source)
+              raise NotImplementedError, "#{name} must implement .extract"
+            end
+
+            def available?
+              return true if gem_name.nil?
+
+              require gem_name
+              true
+            rescue LoadError
+              false
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/data/extract/handlers/csv.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require 'csv'
+
+module Legion
+  module Data
+    module Extract
+      module Handlers
+        class Csv < Base
+          def self.type = :csv
+          def self.extensions = %w[.csv]
+          def self.gem_name = nil
+
+          def self.extract(source)
+            content = source.respond_to?(:read) ? source.read : File.read(source.to_s)
+            table = ::CSV.parse(content, headers: true)
+            text = table.map { |row| row.to_h.map { |k, v| "#{k}: #{v}" }.join(', ') }.join("\n")
+            { text: text, metadata: { rows: table.size, columns: table.headers.size, headers: table.headers } }
+          rescue StandardError => e
+            { text: nil, error: e.message }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/data/extract/handlers/docx.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Legion
+  module Data
+    module Extract
+      module Handlers
+        class Docx < Base
+          def self.type = :docx
+          def self.extensions = %w[.docx]
+          def self.gem_name = 'docx'
+
+          def self.extract(source)
+            require 'docx'
+
+            doc = ::Docx::Document.open(source)
+            paragraphs = doc.paragraphs.map(&:text).reject(&:empty?)
+            text = paragraphs.join("\n\n")
+            { text: text, metadata: { paragraphs: paragraphs.size } }
+          rescue LoadError
+            { text: nil, error: :gem_not_installed, gem: gem_name }
+          rescue StandardError => e
+            { text: nil, error: e.message }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/data/extract/handlers/html.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Legion
+  module Data
+    module Extract
+      module Handlers
+        class Html < Base
+          def self.type = :html
+          def self.extensions = %w[.html .htm]
+          def self.gem_name = 'nokogiri'
+
+          def self.extract(source)
+            require 'nokogiri'
+
+            content = source.respond_to?(:read) ? source.read : File.read(source.to_s)
+            doc = ::Nokogiri::HTML(content)
+
+            # Remove script and style elements
+            doc.css('script, style, noscript').each(&:remove)
+
+            title = doc.at_css('title')&.text&.strip
+            text = doc.text.gsub(/\s+/, ' ').strip
+            { text: text, metadata: { title: title } }
+          rescue LoadError
+            { text: nil, error: :gem_not_installed, gem: gem_name }
+          rescue StandardError => e
+            { text: nil, error: e.message }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/data/extract/handlers/json.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module Legion
+  module Data
+    module Extract
+      module Handlers
+        class Json < Base
+          def self.type = :json
+          def self.extensions = %w[.json]
+          def self.gem_name = nil
+
+          def self.extract(source)
+            content = source.respond_to?(:read) ? source.read : File.read(source.to_s)
+            parsed = ::JSON.parse(content)
+            text = ::JSON.pretty_generate(parsed)
+            { text: text, metadata: { keys: parsed.is_a?(Hash) ? parsed.keys : nil } }
+          rescue StandardError => e
+            { text: nil, error: e.message }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/data/extract/handlers/jsonl.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module Legion
+  module Data
+    module Extract
+      module Handlers
+        class Jsonl < Base
+          def self.type = :jsonl
+          def self.extensions = %w[.jsonl]
+          def self.gem_name = nil
+
+          def self.extract(source)
+            content = source.respond_to?(:read) ? source.read : File.read(source.to_s)
+            lines = content.each_line.map { |l| ::JSON.parse(l.strip) rescue l.strip } # rubocop:disable Style/RescueModifier
+            text = lines.map { |l| l.is_a?(Hash) ? ::JSON.pretty_generate(l) : l }.join("\n---\n")
+            { text: text, metadata: { lines: lines.size } }
+          rescue StandardError => e
+            { text: nil, error: e.message }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/data/extract/handlers/markdown.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Legion
+  module Data
+    module Extract
+      module Handlers
+        class Markdown < Base
+          def self.type = :markdown
+          def self.extensions = %w[.md .markdown]
+          def self.gem_name = nil
+
+          def self.extract(source)
+            content = source.respond_to?(:read) ? source.read : File.read(source.to_s)
+            # Strip YAML frontmatter if present
+            text = content.sub(/\A---\n.*?\n---\n/m, '')
+            { text: text.strip, metadata: { bytes: content.bytesize, has_frontmatter: content != text } }
+          rescue StandardError => e
+            { text: nil, error: e.message }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/data/extract/handlers/pdf.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Legion
+  module Data
+    module Extract
+      module Handlers
+        class Pdf < Base
+          def self.type = :pdf
+          def self.extensions = %w[.pdf]
+          def self.gem_name = 'pdf-reader'
+
+          def self.extract(source)
+            require 'pdf-reader'
+
+            reader = ::PDF::Reader.new(source)
+            text = reader.pages.map(&:text).join("\n\n")
+            { text: text, metadata: { pages: reader.page_count, title: reader.info[:Title] } }
+          rescue LoadError
+            { text: nil, error: :gem_not_installed, gem: gem_name }
+          rescue StandardError => e
+            { text: nil, error: e.message }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/data/extract/handlers/pptx.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Legion
+  module Data
+    module Extract
+      module Handlers
+        class Pptx < Base
+          def self.type = :pptx
+          def self.extensions = %w[.pptx]
+          def self.gem_name = 'rubyzip'
+
+          def self.extract(source)
+            require 'zip'
+            require 'rexml/document'
+
+            slides = []
+            ::Zip::File.open(source) do |zip|
+              zip.glob('ppt/slides/slide*.xml').sort_by(&:name).each do |entry|
+                doc = REXML::Document.new(entry.get_input_stream.read)
+                texts = []
+                doc.each_element('//a:t') { |e| texts << e.text }
+                slides << texts.join(' ') unless texts.empty?
+              end
+            end
+            text = slides.each_with_index.map { |s, i| "Slide #{i + 1}: #{s}" }.join("\n\n")
+            { text: text, metadata: { slides: slides.size } }
+          rescue LoadError
+            { text: nil, error: :gem_not_installed, gem: 'rubyzip' }
+          rescue StandardError => e
+            { text: nil, error: e.message }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/data/extract/handlers/text.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Legion
+  module Data
+    module Extract
+      module Handlers
+        class Text < Base
+          def self.type = :text
+          def self.extensions = %w[.txt]
+          def self.gem_name = nil
+
+          def self.extract(source)
+            content = source.respond_to?(:read) ? source.read : File.read(source.to_s)
+            { text: content, metadata: { bytes: content.bytesize } }
+          rescue StandardError => e
+            { text: nil, error: e.message }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/data/extract/handlers/xlsx.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Legion
+  module Data
+    module Extract
+      module Handlers
+        class Xlsx < Base
+          def self.type = :xlsx
+          def self.extensions = %w[.xlsx .xls]
+          def self.gem_name = 'rubyXL'
+
+          def self.extract(source)
+            require 'rubyXL'
+            require 'rubyXL/convenience_methods'
+
+            workbook = ::RubyXL::Parser.parse(source)
+            sheets = []
+            workbook.worksheets.each do |sheet|
+              rows = sheet.each.map do |row|
+                next unless row
+
+                row.cells.map { |c| c&.value.to_s }.join(', ')
+              end.compact
+              sheets << "Sheet: #{sheet.sheet_name}\n#{rows.join("\n")}" unless rows.empty?
+            end
+            text = sheets.join("\n\n")
+            { text: text, metadata: { sheets: workbook.worksheets.size } }
+          rescue LoadError
+            { text: nil, error: :gem_not_installed, gem: gem_name }
+          rescue StandardError => e
+            { text: nil, error: e.message }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/data/extract/type_detector.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Legion
+  module Data
+    module Extract
+      module TypeDetector
+        EXTENSION_MAP = {
+          '.pdf'      => :pdf,
+          '.docx'     => :docx,
+          '.pptx'     => :pptx,
+          '.xlsx'     => :xlsx,
+          '.xls'      => :xlsx,
+          '.md'       => :markdown,
+          '.markdown' => :markdown,
+          '.txt'      => :text,
+          '.csv'      => :csv,
+          '.json'     => :json,
+          '.jsonl'    => :jsonl,
+          '.html'     => :html,
+          '.htm'      => :html
+        }.freeze
+
+        module_function
+
+        def detect(source)
+          return detect_from_path(source) if source.is_a?(String) && File.exist?(source)
+          return detect_from_io(source) if source.respond_to?(:path)
+
+          nil
+        end
+
+        def detect_from_path(path)
+          ext = File.extname(path).downcase
+          EXTENSION_MAP[ext]
+        end
+
+        def detect_from_io(io)
+          return nil unless io.respond_to?(:path) && io.path
+
+          detect_from_path(io.path)
+        end
+      end
+    end
+  end
+end

data/lib/legion/data/extract.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require_relative 'extract/type_detector'
+require_relative 'extract/handlers/base'
+
+module Legion
+  module Data
+    module Extract
+      class << self
+        def extract(source, type: :auto)
+          detected_type = type == :auto ? TypeDetector.detect(source) : type&.to_sym
+          return { success: false, text: nil, error: :unknown_type } unless detected_type
+
+          handler = Handlers::Base.for_type(detected_type)
+          return { success: false, text: nil, error: :no_handler, type: detected_type } unless handler
+
+          unless handler.available?
+            return { success: false, text: nil, error: :gem_not_installed,
+                     gem: handler.gem_name, type: detected_type }
+          end
+
+          result = handler.extract(source)
+          if result[:text]
+            { success: true, text: result[:text], metadata: result[:metadata], type: detected_type }
+          else
+            { success: false, text: nil, error: result[:error], type: detected_type }
+          end
+        rescue StandardError => e
+          { success: false, text: nil, error: e.message, type: detected_type }
+        end
+
+        def supported_types
+          load_all_handlers
+          Handlers::Base.supported_types
+        end
+
+        def can_extract?(type)
+          load_all_handlers
+          handler = Handlers::Base.for_type(type&.to_sym)
+          handler&.available? || false
+        end
+
+        def register_handler(type, klass)
+          Handlers::Base.registry[type.to_sym] = klass
+        end
+
+        private
+
+        def load_all_handlers
+          return if @handlers_loaded
+
+          Dir[File.join(__dir__, 'extract', 'handlers', '*.rb')].each do |f|
+            require f unless f.end_with?('base.rb')
+          end
+          @handlers_loaded = true
+        end
+      end
+    end
+  end
+end

data/lib/legion/data/migrations/047_apollo_knowledge_capture.rb

@@ -0,0 +1,152 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  up do
+    next unless adapter_scheme == :postgres
+
+    # --- Identity columns on apollo_entries ---
+    alter_table(:apollo_entries) do
+      add_column :submitted_by, String, size: 255
+      add_column :submitted_from, String, size: 255
+      add_column :content_hash, String, fixed: true, size: 32
+    end
+
+    # --- apollo_operations table ---
+    run <<~SQL
+      CREATE TABLE IF NOT EXISTS apollo_operations (
+        id              BIGSERIAL PRIMARY KEY,
+        operation       VARCHAR(50)  NOT NULL,
+        actor           VARCHAR(100) NOT NULL,
+        target_type     VARCHAR(50),
+        target_ids      INTEGER[],
+        summary         TEXT,
+        detail          JSONB,
+        old_state       JSONB,
+        new_state       JSONB,
+        reason          TEXT,
+        principal_id    VARCHAR(255),
+        created_at      TIMESTAMPTZ NOT NULL DEFAULT NOW()
+      );
+    SQL
+
+    # --- apollo_entries_archive table ---
+    run <<~SQL
+      CREATE TABLE IF NOT EXISTS apollo_entries_archive (
+        LIKE apollo_entries INCLUDING ALL,
+        archived_at     TIMESTAMPTZ DEFAULT NOW(),
+        archive_reason  TEXT
+      );
+    SQL
+
+    # --- Indexes: apollo_entries ---
+    run 'CREATE INDEX IF NOT EXISTS idx_apollo_submitted_by ON apollo_entries (submitted_by);'
+    run 'CREATE INDEX IF NOT EXISTS idx_apollo_submitted_from ON apollo_entries (submitted_from);'
+
+    # Content hash dedup (unique among active entries only)
+    run <<~SQL
+      CREATE UNIQUE INDEX IF NOT EXISTS idx_apollo_content_hash
+        ON apollo_entries (content_hash)
+        WHERE status != 'archived';
+    SQL
+
+    # Status filtering (every read query filters on status)
+    run 'CREATE INDEX IF NOT EXISTS idx_apollo_status ON apollo_entries (status);'
+
+    # Partial index: active entries only (hot path)
+    run <<~SQL
+      CREATE INDEX IF NOT EXISTS idx_apollo_active
+        ON apollo_entries (id)
+        WHERE status IN ('candidate', 'confirmed', 'disputed');
+    SQL
+
+    # Confidence ranking and decay targeting
+    run 'CREATE INDEX IF NOT EXISTS idx_apollo_confidence ON apollo_entries (confidence);'
+
+    # Time-based: decay age, archival sweep
+    run 'CREATE INDEX IF NOT EXISTS idx_apollo_created ON apollo_entries (created_at);'
+    run 'CREATE INDEX IF NOT EXISTS idx_apollo_updated ON apollo_entries (updated_at);'
+
+    # Composite: decay cycle targets
+    run <<~SQL
+      CREATE INDEX IF NOT EXISTS idx_apollo_decay_target
+        ON apollo_entries (updated_at)
+        WHERE status != 'archived';
+    SQL
+
+    # Composite: corroboration targets
+    run <<~SQL
+      CREATE INDEX IF NOT EXISTS idx_apollo_candidates
+        ON apollo_entries (status, source_provider, source_channel)
+        WHERE status = 'candidate' AND embedding IS NOT NULL;
+    SQL
+
+    # Knowledge domain (expertise, RBAC)
+    run 'CREATE INDEX IF NOT EXISTS idx_apollo_domain ON apollo_entries (knowledge_domain);'
+
+    # Source agent (expertise aggregation)
+    run 'CREATE INDEX IF NOT EXISTS idx_apollo_source_agent ON apollo_entries (source_agent);'
+
+    # Drop existing HNSW index and recreate as partial (active entries only)
+    run 'DROP INDEX IF EXISTS apollo_entries_embedding_idx;'
+    run <<~SQL
+      CREATE INDEX IF NOT EXISTS idx_apollo_embedding_active
+        ON apollo_entries USING hnsw (embedding vector_cosine_ops)
+        WITH (m = 16, ef_construction = 64)
+        WHERE status IN ('candidate', 'confirmed', 'disputed');
+    SQL
+
+    # --- Indexes: apollo_relations ---
+    run 'CREATE INDEX IF NOT EXISTS idx_apollo_rel_from ON apollo_relations (from_entry_id);'
+    run 'CREATE INDEX IF NOT EXISTS idx_apollo_rel_to ON apollo_relations (to_entry_id);'
+    run 'CREATE INDEX IF NOT EXISTS idx_apollo_rel_type ON apollo_relations (relation_type);'
+    run 'CREATE INDEX IF NOT EXISTS idx_apollo_rel_composite ON apollo_relations (from_entry_id, relation_type);'
+
+    # --- Indexes: apollo_expertise ---
+    run 'CREATE INDEX IF NOT EXISTS idx_apollo_exp_agent ON apollo_expertise (agent_id);'
+    run 'CREATE INDEX IF NOT EXISTS idx_apollo_exp_domain ON apollo_expertise (domain);'
+    run 'CREATE INDEX IF NOT EXISTS idx_apollo_exp_composite ON apollo_expertise (agent_id, domain);'
+
+    # --- Indexes: apollo_operations ---
+    run 'CREATE INDEX IF NOT EXISTS idx_apollo_ops_created ON apollo_operations (created_at);'
+    run 'CREATE INDEX IF NOT EXISTS idx_apollo_ops_operation ON apollo_operations (operation);'
+    run 'CREATE INDEX IF NOT EXISTS idx_apollo_ops_actor ON apollo_operations (actor);'
+    run 'CREATE INDEX IF NOT EXISTS idx_apollo_ops_target ON apollo_operations USING GIN (target_ids);'
+
+    # --- Indexes: apollo_entries_archive ---
+    run 'CREATE INDEX IF NOT EXISTS idx_archive_content_hash ON apollo_entries_archive (content_hash);'
+    run 'CREATE INDEX IF NOT EXISTS idx_archive_source_agent ON apollo_entries_archive (source_agent);'
+    run 'CREATE INDEX IF NOT EXISTS idx_archive_archived_at ON apollo_entries_archive (archived_at);'
+  end
+
+  down do
+    next unless adapter_scheme == :postgres
+
+    # Restore original HNSW index (non-partial)
+    run 'DROP INDEX IF EXISTS idx_apollo_embedding_active;'
+    run <<~SQL
+      CREATE INDEX IF NOT EXISTS apollo_entries_embedding_idx
+        ON apollo_entries USING hnsw (embedding vector_cosine_ops);
+    SQL
+
+    drop_table?(:apollo_entries_archive)
+    drop_table?(:apollo_operations)
+
+    # Drop new indexes
+    %w[
+      idx_apollo_submitted_by idx_apollo_submitted_from idx_apollo_content_hash
+      idx_apollo_status idx_apollo_active idx_apollo_confidence
+      idx_apollo_created idx_apollo_updated idx_apollo_decay_target
+      idx_apollo_candidates idx_apollo_domain idx_apollo_source_agent
+      idx_apollo_rel_from idx_apollo_rel_to idx_apollo_rel_type idx_apollo_rel_composite
+      idx_apollo_exp_agent idx_apollo_exp_domain idx_apollo_exp_composite
+      idx_apollo_ops_created idx_apollo_ops_operation idx_apollo_ops_actor idx_apollo_ops_target
+      idx_archive_content_hash idx_archive_source_agent idx_archive_archived_at
+    ].each { |idx| run "DROP INDEX IF EXISTS #{idx};" }
+
+    alter_table(:apollo_entries) do
+      drop_column :content_hash
+      drop_column :submitted_from
+      drop_column :submitted_by
+    end
+  end
+end

data/lib/legion/data/version.rb

@@ -2,6 +2,6 @@
 
 module Legion
   module Data
-    VERSION = '1.6.3'
+    VERSION = '1.6.6'
   end
 end

data/lib/legion/data.rb

@@ -13,6 +13,7 @@ require_relative 'data/partition_manager'
 require_relative 'data/archiver'
 require_relative 'data/helper'
 require_relative 'data/rls'
+require_relative 'data/extract'
 
 module Legion
   module Data
@@ -55,6 +56,51 @@ module Legion
         }
       end
 
+      def connected?
+        Legion::Settings[:data][:connected] == true
+      rescue StandardError
+        false
+      end
+
+      def can_write?(table_name)
+        return false unless connected?
+
+        adapter = Legion::Settings[:data][:adapter]&.to_s
+        return true if adapter == 'sqlite'
+
+        @write_privileges ||= {}
+        return @write_privileges[table_name] unless @write_privileges[table_name].nil?
+
+        @write_privileges[table_name] = connection
+                                        .fetch("SELECT has_table_privilege(current_user, ?, 'INSERT') AS can", table_name.to_s)
+                                        .first[:can] == true
+      rescue StandardError
+        @write_privileges[table_name] = false if @write_privileges
+        false
+      end
+
+      def can_read?(table_name)
+        return false unless connected?
+
+        adapter = Legion::Settings[:data][:adapter]&.to_s
+        return true if adapter == 'sqlite'
+
+        @read_privileges ||= {}
+        return @read_privileges[table_name] unless @read_privileges[table_name].nil?
+
+        @read_privileges[table_name] = connection
+                                       .fetch("SELECT has_table_privilege(current_user, ?, 'SELECT') AS can", table_name.to_s)
+                                       .first[:can] == true
+      rescue StandardError
+        @read_privileges[table_name] = false if @read_privileges
+        false
+      end
+
+      def reset_privileges!
+        @write_privileges = nil
+        @read_privileges = nil
+      end
+
       def setup_cache
         cache_settings = Legion::Settings[:data][:cache]
         setup_static_cache if cache_settings[:static_cache]

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: legion-data
 version: !ruby/object:Gem::Version
-  version: 1.6.3
+  version: 1.6.6
 platform: ruby
 authors:
 - Esity
@@ -9,6 +9,20 @@ bindir: bin
 cert_chain: []
 date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: csv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.2'
 - !ruby/object:Gem::Dependency
   name: legion-logging
   requirement: !ruby/object:Gem::Requirement
@@ -98,6 +112,19 @@ files:
 - lib/legion/data/encryption/sequel_plugin.rb
 - lib/legion/data/event_store.rb
 - lib/legion/data/event_store/projection.rb
+- lib/legion/data/extract.rb
+- lib/legion/data/extract/handlers/base.rb
+- lib/legion/data/extract/handlers/csv.rb
+- lib/legion/data/extract/handlers/docx.rb
+- lib/legion/data/extract/handlers/html.rb
+- lib/legion/data/extract/handlers/json.rb
+- lib/legion/data/extract/handlers/jsonl.rb
+- lib/legion/data/extract/handlers/markdown.rb
+- lib/legion/data/extract/handlers/pdf.rb
+- lib/legion/data/extract/handlers/pptx.rb
+- lib/legion/data/extract/handlers/text.rb
+- lib/legion/data/extract/handlers/xlsx.rb
+- lib/legion/data/extract/type_detector.rb
 - lib/legion/data/helper.rb
 - lib/legion/data/local.rb
 - lib/legion/data/migration.rb
@@ -147,6 +174,7 @@ files:
 - lib/legion/data/migrations/044_expand_memory_traces.rb
 - lib/legion/data/migrations/045_add_memory_associations.rb
 - lib/legion/data/migrations/046_add_metering_hourly_rollup.rb
+- lib/legion/data/migrations/047_apollo_knowledge_capture.rb
 - lib/legion/data/model.rb
 - lib/legion/data/models/apollo_access_log.rb
 - lib/legion/data/models/apollo_entry.rb