smart_brain 0.2.0 → 0.3.0

data/lib/smart_brain/adapters/smart_rag/media_metadata_extractor.rb

@@ -0,0 +1,255 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'open3'
+
+module SmartBrain
+  module Adapters
+    module SmartRag
+      # Extracts technical metadata from media files (image / audio / video).
+      #
+      # Stage 1 of the media memory plan (docs/media_memory_schema.md):
+      #   - image  -> Python Pillow bridge (same pattern as MarkitdownBridge)
+      #   - audio  -> ffprobe
+      #   - video  -> ffprobe
+      #
+      # Degradation policy (D5): when a dependency is missing the extractor
+      # returns the default media object with a warning instead of raising,
+      # so ingestion can still store file-level metadata.
+      #
+      # NOTE: This component lives in smart_brain during stage 1 (testable
+      # without a SmartRAG database). Stage 3 will sync it into the SmartRAG
+      # project as SmartRAG::Core::MediaMetadataExtractor and wire it into
+      # DocumentProcessor#extract_metadata.
+      class MediaMetadataExtractor
+        IMAGE_EXTS = %w[.jpg .jpeg .png .gif .webp .bmp .tif .tiff .heic .heif].freeze
+        AUDIO_EXTS = %w[.mp3 .wav .m4a .aac .flac .ogg .oga .opus .wma .aif .aiff].freeze
+        VIDEO_EXTS = %w[.mp4 .mkv .mov .webm .avi .flv .wmv .m4v .ts .mpeg .mpg].freeze
+
+        # Wide schema shared by all three media types; type-specific fields
+        # stay nil (design decision D1).
+        DEFAULT_MEDIA = {
+          format: nil,
+          width: nil,
+          height: nil,
+          dpi: nil,
+          color_mode: nil,
+          orientation: nil,
+          exif: {},
+          duration_ms: nil,
+          bitrate_bps: nil,
+          sample_rate_hz: nil,
+          channels: nil,
+          codec: nil,
+          fps: nil,
+          audio_codec: nil
+        }.freeze
+
+        # Extract technical metadata from a media file.
+        # @param [String] file_path path to the media file
+        # @param [String, Symbol, nil] media_type explicit type override
+        # @return [Hash] { media_type:, media: {...}, warnings: [...] }
+        def extract(file_path, media_type: nil)
+          detected = media_type || detect_media_type(file_path)
+          return { media_type: 'other', media: DEFAULT_MEDIA.dup, warnings: [] } unless detected
+
+          type = detected.to_sym
+          media = DEFAULT_MEDIA.dup
+          warnings = []
+
+          case type
+          when :image then extract_image(file_path, media, warnings)
+          when :audio then extract_audio(file_path, media, warnings)
+          when :video then extract_video(file_path, media, warnings)
+          end
+
+          { media_type: type.to_s, media: media, warnings: warnings }
+        rescue StandardError => e
+          {
+            media_type: (type ? type.to_s : 'other'),
+            media: (defined?(media) && media) || DEFAULT_MEDIA.dup,
+            warnings: ["media metadata extraction failed: #{e.message}"]
+          }
+        end
+
+        # Detect media type from file extension.
+        # @param [String] file_path
+        # @return [Symbol, nil] :image | :audio | :video | nil (unknown)
+        def detect_media_type(file_path)
+          ext = File.extname(file_path.to_s).downcase
+          return :image if IMAGE_EXTS.include?(ext)
+          return :audio if AUDIO_EXTS.include?(ext)
+          return :video if VIDEO_EXTS.include?(ext)
+
+          nil
+        end
+
+        # Check whether the image dependency (Pillow) is available.
+        def pillow_available?
+          return @pillow_available unless @pillow_available.nil?
+
+          @pillow_available = system('python3', '-c', 'import PIL', out: File::NULL, err: File::NULL) ||
+                              system('python', '-c', 'import PIL', out: File::NULL, err: File::NULL)
+        end
+
+        # Check whether ffprobe is available.
+        def ffprobe_available?
+          return @ffprobe_available unless @ffprobe_available.nil?
+
+          @ffprobe_available = !ffprobe_cmd.nil?
+        end
+
+        private
+
+        def extract_image(file_path, media, warnings)
+          unless pillow_available?
+            warnings << 'Pillow (PIL) not available; image metadata skipped'
+            return media
+          end
+
+          python = %w[python3 python].find { |cmd| system(cmd, '-c', 'import PIL', out: File::NULL, err: File::NULL) }
+          script = <<~PYTHON
+            import json, sys
+            from PIL import Image
+
+            path = sys.argv[1]
+            img = Image.open(path)
+            info = {
+              'format': (img.format or '').lower() or None,
+              'width': img.width,
+              'height': img.height,
+              'color_mode': img.mode,
+            }
+            dpi = img.info.get('dpi')
+            info['dpi'] = round(dpi[0]) if dpi and isinstance(dpi[0], (int, float))
+            exif = {}
+            try:
+                from PIL import ExifTags
+                raw = img._getexif() if hasattr(img, '_getexif') else None
+                if raw:
+                    for k, v in raw.items():
+                        name = ExifTags.TAGS.get(k, str(k))
+                        if v is None or isinstance(v, (int, float, str)):
+                            exif[name] = v
+            except Exception:
+                pass
+            info['exif'] = exif
+            print(json.dumps(info))
+          PYTHON
+
+          output, status = Open3.capture2e(python, '-c', script, file_path)
+          unless status.success?
+            warnings << "image metadata extraction failed: #{output.strip}"
+            return media
+          end
+
+          parsed = JSON.parse(output)
+          media[:format] = parsed['format']
+          media[:width] = parsed['width']
+          media[:height] = parsed['height']
+          media[:color_mode] = parsed['color_mode']
+          media[:dpi] = parsed['dpi']
+          media[:exif] = parsed['exif'] || {}
+          media[:orientation] = derive_orientation(media[:width], media[:height])
+          media
+        rescue JSON::ParserError, SystemCallError => e
+          warnings << "image metadata extraction failed: #{e.message}"
+          media
+        end
+
+        def extract_audio(file_path, media, warnings)
+          probe = probe_ffprobe(file_path, warnings)
+          return media unless probe
+
+          format_info = probe['format'] || {}
+          stream = probe['streams'].to_a.find { |s| s['codec_type'] == 'audio' } || probe['streams'].to_a.first
+
+          media[:format] = File.extname(file_path.to_s).downcase.sub('.', '')
+          media[:codec] = stream && stream['codec_name']
+          media[:duration_ms] = ms_of(format_info['duration'])
+          media[:bitrate_bps] = int_of(format_info['bit_rate']) || (stream && int_of(stream['bit_rate']))
+          media[:sample_rate_hz] = stream && int_of(stream['sample_rate'])
+          media[:channels] = stream && int_of(stream['channels'])
+          media
+        end
+
+        def extract_video(file_path, media, warnings)
+          probe = probe_ffprobe(file_path, warnings)
+          return media unless probe
+
+          format_info = probe['format'] || {}
+          streams = probe['streams'].to_a
+          video_stream = streams.find { |s| s['codec_type'] == 'video' }
+          audio_stream = streams.find { |s| s['codec_type'] == 'audio' }
+
+          media[:format] = File.extname(file_path.to_s).downcase.sub('.', '')
+          media[:duration_ms] = ms_of(format_info['duration'])
+          media[:bitrate_bps] = int_of(format_info['bit_rate'])
+          if video_stream
+            media[:width] = int_of(video_stream['width'])
+            media[:height] = int_of(video_stream['height'])
+            media[:codec] = video_stream['codec_name']
+            media[:fps] = fps_of(video_stream)
+          end
+          media[:audio_codec] = audio_stream && audio_stream['codec_name']
+          media[:orientation] = derive_orientation(media[:width], media[:height])
+          media
+        end
+
+        def probe_ffprobe(file_path, warnings)
+          cmd = ffprobe_cmd
+          unless cmd
+            warnings << 'ffprobe not available; audio/video metadata skipped'
+            return nil
+          end
+
+          args = [cmd, '-v', 'quiet', '-print_format', 'json', '-show_format', '-show_streams', file_path]
+          output, status, = Open3.capture3(*args)
+          unless status.success?
+            warnings << "ffprobe failed: #{output.strip}"
+            return nil
+          end
+
+          JSON.parse(output)
+        rescue JSON::ParserError, SystemCallError, Errno::ENOENT => e
+          warnings << "ffprobe failed: #{e.message}"
+          nil
+        end
+
+        def ffprobe_cmd
+          @ffprobe_cmd ||= %w[ffprobe].find { |c| system('which', c, out: File::NULL, err: File::NULL) }
+        end
+
+        def ms_of(value)
+          return nil if value.nil?
+
+          (value.to_f * 1000).round
+        end
+
+        def int_of(value)
+          return nil if value.nil?
+
+          Integer(value.to_s, exception: false)
+        end
+
+        def fps_of(stream)
+          rate = stream['avg_frame_rate'].to_s
+          return nil if rate.empty?
+
+          num, den = rate.split('/')
+          return nil if num.nil?
+
+          den = '1' if den.to_s.empty? || den.to_i.zero?
+          (num.to_f / den.to_f).round(2)
+        end
+
+        def derive_orientation(width, height)
+          return nil unless width && height
+          return 'square' if width == height
+
+          width > height ? 'landscape' : 'portrait'
+        end
+      end
+    end
+  end
+end

data/lib/smart_brain/adapters/smart_rag/null_client.rb

@@ -4,6 +4,8 @@ module SmartBrain
   module Adapters
     module SmartRag
       class NullClient
+        WRITE_OPERATIONS = %i[document media image audio video].freeze
+
         def retrieve(plan)
           business_scoped = Array(plan.dig(:scope_context, :read)).any? do |ref|
             (ref[:type] || ref['type']).to_s != 'session'
@@ -21,6 +23,41 @@ module SmartBrain
             scope_filter_applied: !business_scoped
           }
         end
+
+        WRITE_OPERATIONS.each do |type|
+          define_method("add_#{type}") do |_source, _options = {}|
+            { status: 'unsupported', media_type: type.to_s, section_count: 0,
+              warnings: ["smart_rag client not configured; add_#{type} ignored"] }
+          end
+        end
+
+        def enqueue_media(_source, _options = {})
+          { status: 'unsupported', warnings: ['smart_rag client not configured; enqueue_media ignored'] }
+        end
+
+        def media_job(job_id)
+          { status: 'unsupported', job_id: job_id, warnings: ['smart_rag client not configured; media_job ignored'] }
+        end
+
+        def media_jobs(status: nil, limit: 20, offset: 0)
+          { status: 'unsupported', jobs: [], total: 0,
+            warnings: ['smart_rag client not configured; media_jobs ignored'] }
+        end
+
+        def cancel_media_job(job_id)
+          { status: 'unsupported', job_id: job_id,
+            warnings: ['smart_rag client not configured; cancel_media_job ignored'] }
+        end
+
+        def retry_media_job(job_id)
+          { status: 'unsupported', job_id: job_id,
+            warnings: ['smart_rag client not configured; retry_media_job ignored'] }
+        end
+
+        def media_job_statistics
+          { status: 'unsupported', counts: {}, total: 0,
+            warnings: ['smart_rag client not configured; media_job_statistics ignored'] }
+        end
       end
     end
   end

data/lib/smart_brain/configuration.rb

@@ -85,7 +85,10 @@ module SmartBrain
     def database_config
       cfg = (storage.fetch(:database, {}) || {}).dup
       %i[host port database user password].each do |key|
-        env_key = "SMARTBRAIN_DB_#{key.to_s.upcase}"
+        # The `database` key maps to SMARTBRAIN_DB_NAME to match the documented
+        # env contract (SMARTBRAIN_DB_{HOST,PORT,NAME,USER,PASSWORD}).
+        suffix = key == :database ? 'NAME' : key.to_s.upcase
+        env_key = "SMARTBRAIN_DB_#{suffix}"
         cfg[key] = ENV[env_key] if ENV.key?(env_key)
       end
       cfg

data/lib/smart_brain/runtime.rb

@@ -264,7 +264,8 @@ module SmartBrain
       result
     end
 
-    def compose_context(session_id:, user_message:, agent_state: {}, domain_id: nil, scope_context: nil)
+    def compose_context(session_id:, user_message:, agent_state: {}, domain_id: nil, scope_context: nil,
+                        resource_filters: {})
       started_at = monotonic_time
       resolved = resolve_scopes(domain_id: domain_id, session_id: session_id, scope_context: scope_context)
       recent_turns = event_store.recent_turns(session_id: session_id, limit: config.composition.fetch(:recent_turns_max, 8))
@@ -279,6 +280,7 @@ module SmartBrain
         refs: refs
       )
       plan[:scope_context] = public_scope_context(resolved)
+      plan[:global_filters] = plan.fetch(:global_filters, {}).merge(symbolize_keys(resource_filters))
       plan[:debug] = (plan[:debug] || {}).merge(
         caller: { app: 'smart_brain', domain_id: resolved[:domain_id], session_id: session_id }
       )
@@ -363,6 +365,18 @@ module SmartBrain
       conflict_resolver.resolve(result)[:selected].first(limit)
     end
 
+    def add_document(source:, options: {}) = smart_rag_client.add_document(source, options)
+    def add_media(source:, options: {}) = smart_rag_client.add_media(source, options)
+    def add_image(source:, options: {}) = smart_rag_client.add_image(source, options)
+    def add_audio(source:, options: {}) = smart_rag_client.add_audio(source, options)
+    def add_video(source:, options: {}) = smart_rag_client.add_video(source, options)
+    def enqueue_media(source:, options: {}) = smart_rag_client.enqueue_media(source, options)
+    def media_job(job_id:) = smart_rag_client.media_job(job_id)
+    def media_jobs(status: nil, limit: 20, offset: 0) = smart_rag_client.media_jobs(status: status, limit: limit, offset: offset)
+    def cancel_media_job(job_id:) = smart_rag_client.cancel_media_job(job_id)
+    def retry_media_job(job_id:) = smart_rag_client.retry_media_job(job_id)
+    def media_job_statistics = smart_rag_client.media_job_statistics
+
     def memory_item(id:)
       memory_store.find_item(id: id)
     end
@@ -413,6 +427,10 @@ module SmartBrain
       raise
     end
 
+    def symbolize_keys(hash)
+      hash.each_with_object({}) { |(key, value), result| result[key.to_sym] = value }
+    end
+
     def scope_validation_failure!(message)
       error = ArgumentError.new(message)
       tracker.log_scope_validation_failure(error)

data/lib/smart_brain/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SmartBrain
-  VERSION = '0.2.0'
+  VERSION = '0.3.0'
 end

data/lib/smart_brain.rb

@@ -18,13 +18,26 @@ module SmartBrain
       runtime.commit_turn(session_id: session_id, turn_events: turn_events, domain_id: domain_id, scope_context: scope_context)
     end
 
-    def compose_context(session_id:, user_message:, agent_state: {}, domain_id: nil, scope_context: nil)
+    def compose_context(session_id:, user_message:, agent_state: {}, domain_id: nil, scope_context: nil,
+                        resource_filters: {})
       runtime.compose_context(
         session_id: session_id, user_message: user_message, agent_state: agent_state,
-        domain_id: domain_id, scope_context: scope_context
+        domain_id: domain_id, scope_context: scope_context, resource_filters: resource_filters
       )
     end
 
+    def add_document(source:, options: {}) = runtime.add_document(source: source, options: options)
+    def add_media(source:, options: {}) = runtime.add_media(source: source, options: options)
+    def add_image(source:, options: {}) = runtime.add_image(source: source, options: options)
+    def add_audio(source:, options: {}) = runtime.add_audio(source: source, options: options)
+    def add_video(source:, options: {}) = runtime.add_video(source: source, options: options)
+    def enqueue_media(source:, options: {}) = runtime.enqueue_media(source: source, options: options)
+    def media_job(job_id:) = runtime.media_job(job_id: job_id)
+    def media_jobs(status: nil, limit: 20, offset: 0) = runtime.media_jobs(status: status, limit: limit, offset: offset)
+    def cancel_media_job(job_id:) = runtime.cancel_media_job(job_id: job_id)
+    def retry_media_job(job_id:) = runtime.retry_media_job(job_id: job_id)
+    def media_job_statistics = runtime.media_job_statistics
+
     def search_memory(session_id:, query:, limit: nil, domain_id: nil, scope_context: nil)
       runtime.search_memory(
         session_id: session_id, query: query, limit: limit, domain_id: domain_id, scope_context: scope_context

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: smart_brain
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - SmartBrain Team
@@ -9,34 +9,6 @@ bindir: exe
 cert_chain: []
 date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: dry-struct
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.7'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.7'
-- !ruby/object:Gem::Dependency
-  name: dry-validation
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.11'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.11'
 - !ruby/object:Gem::Dependency
   name: faraday
   requirement: !ruby/object:Gem::Requirement
@@ -52,19 +24,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.11'
 - !ruby/object:Gem::Dependency
-  name: oj
+  name: faraday-multipart
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.16'
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.16'
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: pg
   requirement: !ruby/object:Gem::Requirement
@@ -94,47 +66,47 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '7.2'
 - !ruby/object:Gem::Dependency
-  name: sequel
+  name: rackup
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.87'
+        version: '2.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.87'
+        version: '2.1'
 - !ruby/object:Gem::Dependency
-  name: sinatra
+  name: sequel
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.2'
+        version: '5.87'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.2'
+        version: '5.87'
 - !ruby/object:Gem::Dependency
-  name: smart_rag
+  name: sinatra
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: '4.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: '4.2'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -259,7 +231,9 @@ files:
 - docs/context_package.md
 - docs/evidence_pack.md
 - docs/gap_vs_mempal.md
+- docs/installation.md
 - docs/mcp.md
+- docs/media_memory_schema.md
 - docs/memory_types.md
 - docs/multi_scope_memory_refactor_plan.md
 - docs/multi_scope_migration.md
@@ -274,11 +248,14 @@ files:
 - examples/04_ollama_llm.rb
 - examples/05_smart_rag_integration.rb
 - examples/06_multi_scope_memory.rb
+- examples/07_media_memory.rb
 - examples/README.md
 - exe/smart_brain
 - lib/smart_brain.rb
 - lib/smart_brain/adapters/smart_rag/direct_client.rb
 - lib/smart_brain/adapters/smart_rag/http_client.rb
+- lib/smart_brain/adapters/smart_rag/http_transport.rb
+- lib/smart_brain/adapters/smart_rag/media_metadata_extractor.rb
 - lib/smart_brain/adapters/smart_rag/null_client.rb
 - lib/smart_brain/adapters/smart_rag/scope_filter.rb
 - lib/smart_brain/configuration.rb
@@ -329,6 +306,23 @@ licenses:
 metadata:
   source_code_uri: https://github.com/zhuangbiaowei/smart_brain
   changelog_uri: https://github.com/zhuangbiaowei/smart_brain/blob/master/CHANGELOG.md
+  documentation_uri: https://github.com/zhuangbiaowei/smart_brain/blob/master/docs/user_guide.md
+post_install_message: |
+  SmartBrain 已安装（0.3.0）。默认零外部依赖：memory 后端 + stub LLM。
+
+  快速开始：
+    smart_brain --version
+    smart_brain status
+    ruby -e "require 'smart_brain'; SmartBrain.configure; p SmartBrain.compose_context(session_id: 's', user_message: '你好')"
+
+  常用配置（环境变量）：
+    SMARTBRAIN_BACKEND=postgres          # 持久化（需先建库，或 smart_brain migrate）
+    SMARTBRAIN_DB_NAME=smart_brain_development  # 其余 SMARTBRAIN_DB_{HOST,PORT,USER,PASSWORD}
+    SMARTBRAIN_LLM_PROVIDER=ollama       # 真 LLM 摘要/重排（需本地 ollama + 生成式模型如 qwen3/llama2）
+    SMARTBRAIN_LLM_BASE_URL / SMARTBRAIN_LLM_API_KEY / SMARTBRAIN_LLM_MODEL
+
+  资源 RAG（可选）：gem install smart_rag，再用 SmartBrain::Adapters::SmartRag::DirectClient / HttpClient。
+  详见 README 与 docs/user_guide.md、docs/installation.md。
 rdoc_options: []
 require_paths:
 - lib
@@ -343,7 +337,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.13
+rubygems_version: 4.0.16
 specification_version: 4
 summary: Agent memory runtime and context composer
 test_files: []