broadlistening 0.7.0

data/lib/broadlistening/comment.rb

@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+module Broadlistening
+  # Represents a normalized comment in the pipeline.
+  #
+  # Comments are the input data to the pipeline, containing user opinions
+  # that will be processed into arguments through the extraction step.
+  #
+  # @example Creating from a hash
+  #   comment = Comment.from_hash({ id: "1", body: "I think...", attribute_age: "30代" })
+  #   comment.id        # => "1"
+  #   comment.body      # => "I think..."
+  #   comment.attributes # => { "age" => "30代" }
+  class Comment
+    attr_accessor :id, :body, :proposal_id, :source_url, :attributes, :properties
+
+    def initialize(id:, body:, proposal_id: nil, source_url: nil, attributes: nil, properties: nil)
+      @id = id
+      @body = body
+      @proposal_id = proposal_id
+      @source_url = source_url
+      @attributes = attributes
+      @properties = properties
+    end
+
+    # Create a Comment from a hash, normalizing various input formats
+    #
+    # @param hash [Hash] Input hash with comment data
+    # @param property_names [Array<String>] Property names to extract (from config)
+    # @return [Comment]
+    def self.from_hash(hash, property_names: [])
+      new(
+        id: hash[:id] || hash["id"],
+        body: hash[:body] || hash["body"],
+        proposal_id: hash[:proposal_id] || hash["proposal_id"],
+        source_url: extract_source_url(hash),
+        attributes: extract_attributes(hash),
+        properties: extract_properties(hash, property_names)
+      )
+    end
+
+    # Create a Comment from an object (e.g., ActiveRecord model)
+    #
+    # @param obj [Object] Object responding to id, body, etc.
+    # @param property_names [Array<String>] Property names to extract
+    # @return [Comment]
+    def self.from_object(obj, property_names: [])
+      new(
+        id: obj.id,
+        body: obj.body,
+        proposal_id: obj.respond_to?(:proposal_id) ? obj.proposal_id : nil,
+        source_url: obj.respond_to?(:source_url) ? obj.source_url : nil,
+        attributes: extract_attributes_from_object(obj),
+        properties: extract_properties_from_object(obj, property_names)
+      )
+    end
+
+    # Convert to hash for serialization
+    #
+    # @return [Hash]
+    def to_h
+      {
+        id: @id,
+        body: @body,
+        proposal_id: @proposal_id,
+        source_url: @source_url,
+        attributes: @attributes,
+        properties: @properties
+      }
+    end
+
+    # Check if comment body is empty or nil
+    #
+    # @return [Boolean]
+    def empty?
+      @body.nil? || @body.strip.empty?
+    end
+
+    class << self
+      private
+
+      def extract_source_url(hash)
+        hash[:source_url] || hash["source_url"] ||
+          hash[:"source-url"] || hash["source-url"]
+      end
+
+      def extract_attributes(hash)
+        attributes = {}
+        hash.each do |key, value|
+          key_str = key.to_s
+          next unless key_str.start_with?("attribute_") || key_str.start_with?("attribute-")
+
+          attr_name = key_str.sub(/^attribute[-_]/, "")
+          attributes[attr_name] = value
+        end
+        attributes.empty? ? nil : attributes
+      end
+
+      def extract_attributes_from_object(obj)
+        return nil unless obj.respond_to?(:attributes) && obj.attributes.is_a?(Hash)
+
+        obj.attributes.empty? ? nil : obj.attributes
+      end
+
+      def extract_properties(hash, property_names)
+        return nil if property_names.empty?
+
+        properties = {}
+        property_names.each do |prop_name|
+          value = hash[prop_name.to_sym] || hash[prop_name.to_s]
+          properties[prop_name.to_s] = value
+        end
+        properties.values.all?(&:nil?) ? nil : properties
+      end
+
+      def extract_properties_from_object(obj, property_names)
+        return nil if property_names.empty?
+
+        properties = {}
+        property_names.each do |prop_name|
+          value = obj.respond_to?(prop_name) ? obj.public_send(prop_name) : nil
+          properties[prop_name.to_s] = value
+        end
+        properties.values.all?(&:nil?) ? nil : properties
+      end
+    end
+  end
+end

data/lib/broadlistening/compatibility.rb

@@ -0,0 +1,375 @@
+# frozen_string_literal: true
+
+require "json"
+require "json_schemer"
+
+module Broadlistening
+  # Compatibility utilities for comparing outputs between
+  # Kouchou-AI (Python) and Broadlistening gem (Ruby) implementations.
+  #
+  # @example Compare two output files
+  #   report = Compatibility.compare_outputs(
+  #     python_output: "path/to/python/result.json",
+  #     ruby_output: "path/to/ruby/result.json"
+  #   )
+  #   puts report.summary
+  #
+  # @example Validate output against schema
+  #   errors = Compatibility.validate_output(result_hash)
+  module Compatibility
+    # Expected structure for hierarchical_result.json
+    REQUIRED_TOP_LEVEL_KEYS = %w[arguments clusters comments propertyMap translations overview config].freeze
+    REQUIRED_ARGUMENT_KEYS = %w[arg_id argument comment_id x y cluster_ids].freeze
+    REQUIRED_CLUSTER_KEYS = %w[level id label takeaway value parent].freeze
+
+    # Path to JSON Schema file
+    SCHEMA_PATH = File.expand_path("../../schema/hierarchical_result.json", __dir__)
+
+    class ComparisonReport
+      attr_accessor :differences, :python_stats, :ruby_stats
+
+      def initialize
+        @differences = []
+        @python_stats = {}
+        @ruby_stats = {}
+      end
+
+      def add_difference(category, message, details = {})
+        @differences << {
+          category: category,
+          message: message,
+          details: details
+        }
+      end
+
+      def compatible?
+        @differences.empty?
+      end
+
+      def summary
+        lines = []
+        lines << "=" * 60
+        lines << "Compatibility Report"
+        lines << "=" * 60
+        lines << ""
+
+        lines << "Python Output Stats:"
+        @python_stats.each { |k, v| lines << "  #{k}: #{v}" }
+        lines << ""
+
+        lines << "Ruby Output Stats:"
+        @ruby_stats.each { |k, v| lines << "  #{k}: #{v}" }
+        lines << ""
+
+        if compatible?
+          lines << "Result: COMPATIBLE"
+        else
+          lines << "Result: INCOMPATIBLE (#{@differences.size} differences found)"
+          lines << ""
+          lines << "Differences:"
+          @differences.each_with_index do |diff, i|
+            lines << "  #{i + 1}. [#{diff[:category]}] #{diff[:message]}"
+            diff[:details].each { |k, v| lines << "      #{k}: #{v}" } if diff[:details].any?
+          end
+        end
+
+        lines << ""
+        lines << "=" * 60
+        lines.join("\n")
+      end
+
+      def to_h
+        {
+          compatible: compatible?,
+          python_stats: @python_stats,
+          ruby_stats: @ruby_stats,
+          differences: @differences
+        }
+      end
+    end
+
+    class << self
+      # Compare outputs from Python and Ruby implementations
+      #
+      # @param python_output [String, Hash] Path to JSON file or parsed hash
+      # @param ruby_output [String, Hash] Path to JSON file or parsed hash
+      # @return [ComparisonReport]
+      def compare_outputs(python_output:, ruby_output:)
+        python_data = load_output(python_output)
+        ruby_data = load_output(ruby_output)
+
+        report = ComparisonReport.new
+        report.python_stats = collect_stats(python_data)
+        report.ruby_stats = collect_stats(ruby_data)
+
+        compare_structure(python_data, ruby_data, report)
+        compare_arguments(python_data, ruby_data, report)
+        compare_clusters(python_data, ruby_data, report)
+        compare_overview(python_data, ruby_data, report)
+
+        report
+      end
+
+      # Validate output structure
+      #
+      # @param output [Hash] Parsed output hash
+      # @return [Array<String>] List of validation errors
+      def validate_output(output)
+        errors = []
+
+        # Check top-level keys
+        missing_keys = REQUIRED_TOP_LEVEL_KEYS - output.keys.map(&:to_s)
+        errors << "Missing top-level keys: #{missing_keys.join(', ')}" if missing_keys.any?
+
+        # Check arguments structure
+        if output["arguments"] || output[:arguments]
+          args = output["arguments"] || output[:arguments]
+          if args.is_a?(Array) && args.any?
+            sample = args.first
+            sample_keys = sample.keys.map(&:to_s)
+            missing_arg_keys = REQUIRED_ARGUMENT_KEYS - sample_keys
+            errors << "Missing argument keys: #{missing_arg_keys.join(', ')}" if missing_arg_keys.any?
+          end
+        end
+
+        # Check clusters structure
+        if output["clusters"] || output[:clusters]
+          clusters = output["clusters"] || output[:clusters]
+          if clusters.is_a?(Array) && clusters.any?
+            sample = clusters.first
+            sample_keys = sample.keys.map(&:to_s)
+            missing_cluster_keys = REQUIRED_CLUSTER_KEYS - sample_keys
+            errors << "Missing cluster keys: #{missing_cluster_keys.join(', ')}" if missing_cluster_keys.any?
+          end
+        end
+
+        errors
+      end
+
+      # Check if output is structurally compatible with Kouchou-AI format
+      #
+      # @param output [Hash] Parsed output hash
+      # @return [Boolean]
+      def valid_output?(output)
+        validate_output(output).empty?
+      end
+
+      # Validate output against JSON Schema
+      #
+      # @param output [Hash, String] Parsed output hash or path to JSON file
+      # @return [Array<Hash>] List of validation errors from JSON Schema
+      def validate_with_schema(output)
+        data = output.is_a?(String) ? JSON.parse(File.read(output)) : output
+        data = deep_stringify_keys(data) if data.is_a?(Hash)
+
+        schema = JSONSchemer.schema(Pathname.new(SCHEMA_PATH))
+        errors = schema.validate(data).to_a
+
+        errors.map do |error|
+          {
+            path: error["data_pointer"],
+            message: error["error"],
+            details: error["details"] || {}
+          }
+        end
+      end
+
+      # Check if output is valid according to JSON Schema
+      #
+      # @param output [Hash, String] Parsed output hash or path to JSON file
+      # @return [Boolean]
+      def valid_schema?(output)
+        validate_with_schema(output).empty?
+      end
+
+      # Get the JSON Schema as a Hash
+      #
+      # @return [Hash] The JSON Schema
+      def schema
+        @schema ||= JSON.parse(File.read(SCHEMA_PATH))
+      end
+
+      # Get the path to the JSON Schema file
+      #
+      # @return [String] Path to schema file
+      def schema_path
+        SCHEMA_PATH
+      end
+
+      private
+
+      def load_output(output)
+        case output
+        when String
+          JSON.parse(File.read(output))
+        when Hash
+          deep_stringify_keys(output)
+        else
+          raise ArgumentError, "Output must be a file path (String) or Hash"
+        end
+      end
+
+      def deep_stringify_keys(hash)
+        hash.transform_keys(&:to_s).transform_values do |v|
+          case v
+          when Hash then deep_stringify_keys(v)
+          when Array then v.map { |e| e.is_a?(Hash) ? deep_stringify_keys(e) : e }
+          else v
+          end
+        end
+      end
+
+      def collect_stats(data)
+        {
+          argument_count: data["arguments"]&.size || 0,
+          cluster_count: data["clusters"]&.size || 0,
+          cluster_levels: data["clusters"]&.map { |c| c["level"] }&.uniq&.sort || [],
+          has_overview: !data["overview"].to_s.strip.empty?,
+          has_property_map: data["propertyMap"]&.any? || false,
+          top_level_keys: data.keys.sort
+        }
+      end
+
+      def compare_structure(python, ruby, report)
+        python_keys = python.keys.sort
+        ruby_keys = ruby.keys.sort
+
+        missing_in_ruby = python_keys - ruby_keys
+        extra_in_ruby = ruby_keys - python_keys
+
+        if missing_in_ruby.any?
+          report.add_difference(
+            :structure,
+            "Missing top-level keys in Ruby output",
+            missing: missing_in_ruby
+          )
+        end
+
+        if extra_in_ruby.any?
+          report.add_difference(
+            :structure,
+            "Extra top-level keys in Ruby output",
+            extra: extra_in_ruby
+          )
+        end
+      end
+
+      def compare_arguments(python, ruby, report)
+        python_args = python["arguments"] || []
+        ruby_args = ruby["arguments"] || []
+
+        # Compare argument structure (keys)
+        if python_args.any? && ruby_args.any?
+          python_keys = python_args.first.keys.sort
+          ruby_keys = ruby_args.first.keys.sort
+
+          missing_keys = python_keys - ruby_keys
+          if missing_keys.any?
+            report.add_difference(
+              :arguments,
+              "Missing argument keys in Ruby output",
+              missing: missing_keys
+            )
+          end
+        end
+
+        # Compare cluster_ids format
+        if python_args.any? && ruby_args.any?
+          python_cluster_ids = python_args.first["cluster_ids"]
+          ruby_cluster_ids = ruby_args.first["cluster_ids"]
+
+          if python_cluster_ids.is_a?(Array) && ruby_cluster_ids.is_a?(Array)
+            # Check format consistency (e.g., "0", "1_5", "2_10")
+            python_format = detect_cluster_id_format(python_cluster_ids)
+            ruby_format = detect_cluster_id_format(ruby_cluster_ids)
+
+            if python_format != ruby_format
+              report.add_difference(
+                :arguments,
+                "cluster_ids format mismatch",
+                python_format: python_format,
+                ruby_format: ruby_format
+              )
+            end
+          end
+        end
+      end
+
+      def compare_clusters(python, ruby, report)
+        python_clusters = python["clusters"] || []
+        ruby_clusters = ruby["clusters"] || []
+
+        # Compare cluster structure
+        if python_clusters.any? && ruby_clusters.any?
+          python_keys = python_clusters.first.keys.sort
+          ruby_keys = ruby_clusters.first.keys.sort
+
+          missing_keys = python_keys - ruby_keys
+          if missing_keys.any?
+            report.add_difference(
+              :clusters,
+              "Missing cluster keys in Ruby output",
+              missing: missing_keys
+            )
+          end
+        end
+
+        # Compare hierarchy levels
+        python_levels = python_clusters.map { |c| c["level"] }.uniq.sort
+        ruby_levels = ruby_clusters.map { |c| c["level"] }.uniq.sort
+
+        if python_levels != ruby_levels
+          report.add_difference(
+            :clusters,
+            "Cluster hierarchy levels differ",
+            python_levels: python_levels,
+            ruby_levels: ruby_levels
+          )
+        end
+
+        # Compare root cluster
+        python_root = python_clusters.find { |c| c["level"] == 0 }
+        ruby_root = ruby_clusters.find { |c| c["level"] == 0 }
+
+        if python_root && ruby_root
+          if python_root["id"] != ruby_root["id"]
+            report.add_difference(
+              :clusters,
+              "Root cluster ID differs",
+              python_id: python_root["id"],
+              ruby_id: ruby_root["id"]
+            )
+          end
+        end
+      end
+
+      def compare_overview(python, ruby, report)
+        python_overview = python["overview"].to_s.strip
+        ruby_overview = ruby["overview"].to_s.strip
+
+        if python_overview.empty? != ruby_overview.empty?
+          report.add_difference(
+            :overview,
+            "Overview presence differs",
+            python_has_overview: !python_overview.empty?,
+            ruby_has_overview: !ruby_overview.empty?
+          )
+        end
+      end
+
+      def detect_cluster_id_format(cluster_ids)
+        return :empty if cluster_ids.empty?
+
+        formats = cluster_ids.map do |id|
+          case id.to_s
+          when /^\d+$/ then :numeric
+          when /^\d+_\d+$/ then :level_index
+          else :other
+          end
+        end
+
+        formats.uniq.size == 1 ? formats.first : :mixed
+      end
+    end
+  end
+end

data/lib/broadlistening/config.rb

@@ -0,0 +1,190 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Broadlistening
+  class Config
+    attr_reader :model, :embedding_model, :provider, :cluster_nums, :workers, :prompts, :api_key,
+                :enable_source_link, :hidden_properties, :is_pubcom,
+                :api_base_url, :local_llm_address, :azure_api_version,
+                :input, :question, :name, :intro
+
+    DEFAULT_CLUSTER_NUMS = [ 5, 15 ].freeze
+    DEFAULT_WORKERS = 10
+    DEFAULT_AZURE_API_VERSION = "2024-02-15-preview"
+
+    def self.from_json(json_string)
+      data = JSON.parse(json_string, symbolize_names: true)
+      from_hash(data)
+    end
+
+    def self.from_hash(hash)
+      prompts = hash[:prompts]&.transform_keys(&:to_sym)
+
+      cluster_nums = hash[:cluster_nums] || hash.dig(:hierarchical_clustering, :cluster_nums)
+      workers = hash[:workers] || hash.dig(:extraction, :workers)
+      hidden_properties = hash[:hidden_properties] || hash.dig(:aggregation, :hidden_properties)
+
+      new(
+        api_key: hash[:api_key],
+        model: hash[:model],
+        embedding_model: hash[:embedding_model],
+        provider: hash[:provider],
+        cluster_nums: cluster_nums,
+        workers: workers,
+        prompts: prompts,
+        enable_source_link: hash[:enable_source_link],
+        hidden_properties: hidden_properties,
+        is_pubcom: hash[:is_pubcom],
+        api_base_url: hash[:api_base_url],
+        local_llm_address: hash[:local_llm_address],
+        azure_api_version: hash[:azure_api_version],
+        input: hash[:input],
+        question: hash[:question],
+        name: hash[:name],
+        intro: hash[:intro]
+      )
+    end
+
+    def self.from_file(path)
+      from_json(File.read(path))
+    end
+
+    def initialize(options = {})
+      @local_llm_address = options[:local_llm_address] || ENV.fetch("LOCAL_LLM_ADDRESS", "localhost:11434")
+      @provider_obj = Provider.new(
+        options[:provider]&.to_sym || :openai,
+        local_llm_address: @local_llm_address
+      )
+      @provider = @provider_obj.name
+      @model = options[:model] || @provider_obj.default_model
+      @embedding_model = options[:embedding_model] || @provider_obj.default_embedding_model
+      @cluster_nums = options[:cluster_nums] || DEFAULT_CLUSTER_NUMS.dup
+      @workers = options[:workers] || DEFAULT_WORKERS
+      @prompts = default_prompts.merge(options[:prompts] || {})
+      @api_key = options[:api_key] || @provider_obj.api_key
+      @enable_source_link = options.fetch(:enable_source_link, false)
+      @hidden_properties = options.fetch(:hidden_properties, {}) || {}
+      @is_pubcom = options.fetch(:is_pubcom, false)
+      @api_base_url = options[:api_base_url] || @provider_obj.base_url
+      @azure_api_version = options[:azure_api_version] || ENV.fetch("AZURE_API_VERSION", DEFAULT_AZURE_API_VERSION)
+      @input = options[:input]
+      @question = options[:question]
+      @name = options[:name]
+      @intro = options[:intro]
+
+      validate!
+    end
+
+    def to_h
+      {
+        model: model,
+        embedding_model: embedding_model,
+        provider: provider,
+        cluster_nums: cluster_nums,
+        workers: workers,
+        enable_source_link: enable_source_link,
+        hidden_properties: hidden_properties,
+        is_pubcom: is_pubcom,
+        api_base_url: api_base_url,
+        local_llm_address: local_llm_address,
+        azure_api_version: azure_api_version,
+        input: input,
+        question: question,
+        name: name,
+        intro: intro
+      }.compact
+    end
+
+    def to_json(*args)
+      to_h.merge(prompts: prompts).to_json(*args)
+    end
+
+    def save_to_file(path)
+      File.write(path, JSON.pretty_generate(to_h.merge(prompts: prompts)))
+    end
+
+    def property_names
+      hidden_properties.keys
+    end
+
+    private
+
+    def validate!
+      if @provider_obj.requires_api_key? && (api_key.nil? || api_key.empty?)
+        raise ConfigurationError, "API key is required"
+      end
+      if @provider_obj.requires_base_url? && (api_base_url.nil? || api_base_url.empty?)
+        raise ConfigurationError, "Azure requires api_base_url"
+      end
+      raise ConfigurationError, "cluster_nums must have at least 2 levels" if cluster_nums.size < 2
+      raise ConfigurationError, "cluster_nums must be sorted ascending" unless cluster_nums == cluster_nums.sort
+    end
+
+    def default_prompts
+      {
+        extraction: extraction_prompt,
+        initial_labelling: initial_labelling_prompt,
+        merge_labelling: merge_labelling_prompt,
+        overview: overview_prompt
+      }
+    end
+
+    def extraction_prompt
+      <<~PROMPT
+        あなたは意見抽出の専門家です。
+        以下のコメントから、主要な意見や主張を抽出してください。
+        1つのコメントに複数の意見が含まれる場合は、それぞれを別々に抽出してください。
+        抽出した意見はJSON形式で返してください。
+
+        出力フォーマット:
+        {"extractedOpinionList": ["意見1", "意見2", ...]}
+
+        注意:
+        - 事実の記述ではなく、意見や主張を抽出してください
+        - 曖昧な表現は具体的に言い換えてください
+        - 重複する意見は1つにまとめてください
+      PROMPT
+    end
+
+    def initial_labelling_prompt
+      <<~PROMPT
+        あなたはクラスタ分析の専門家です。
+        以下の意見グループに対して、適切なラベルと説明を付けてください。
+
+        出力フォーマット:
+        {"label": "ラベル名", "description": "このグループの説明"}
+
+        注意:
+        - ラベルは簡潔で分かりやすいものにしてください（10文字以内推奨）
+        - 説明はグループの特徴を端的に表してください（50文字以内推奨）
+      PROMPT
+    end
+
+    def merge_labelling_prompt
+      <<~PROMPT
+        あなたはクラスタ分析の専門家です。
+        以下の子クラスタのラベルと説明を統合し、親クラスタのラベルと説明を作成してください。
+
+        出力フォーマット:
+        {"label": "ラベル名", "description": "このグループの説明"}
+
+        注意:
+        - 親ラベルは子ラベルの共通テーマを表すものにしてください
+        - 抽象度を上げすぎず、具体性を保ってください
+      PROMPT
+    end
+
+    def overview_prompt
+      <<~PROMPT
+        あなたは分析レポートの専門家です。
+        以下のクラスタ分析結果に基づいて、全体の概要を作成してください。
+
+        注意:
+        - 主要なテーマや傾向を簡潔にまとめてください
+        - 200-300文字程度で記述してください
+        - 客観的な記述を心がけてください
+      PROMPT
+    end
+  end
+end