broadlistening 0.7.0

data/lib/broadlistening/context.rb

@@ -0,0 +1,180 @@
+# frozen_string_literal: true
+
+require "pathname"
+require "fileutils"
+require "json"
+
+module Broadlistening
+  # Manages pipeline execution context - all data flowing through the pipeline.
+  #
+  # The Context holds all intermediate results and provides methods for
+  # loading from / saving to disk for incremental execution support.
+  #
+  # @example Creating a new context
+  #   context = Context.new
+  #   context.comments = [Comment.new(...), ...]
+  #   context.save_step(:extraction, output_dir)
+  #
+  # @example Loading from existing output
+  #   context = Context.load_from_dir("/path/to/output")
+  class Context
+    attr_accessor :comments, :arguments, :relations,
+                  :cluster_results, :umap_coords,
+                  :initial_labels, :labels, :overview, :result,
+                  :output_dir
+
+    # Output file mapping for each step
+    OUTPUT_FILES = {
+      extraction: "extraction.json",
+      embedding: "embeddings.json",
+      clustering: "clustering.json",
+      initial_labelling: "initial_labels.json",
+      merge_labelling: "merge_labels.json",
+      overview: "overview.json",
+      aggregation: "result.json"
+    }.freeze
+
+    # Load existing context from output directory
+    #
+    # @param output_dir [String, Pathname] Directory containing output files
+    # @return [Context] A new context populated with data from output files
+    def self.load_from_dir(output_dir)
+      context = new
+      dir = Pathname.new(output_dir)
+
+      OUTPUT_FILES.each do |step, filename|
+        file = dir / filename
+        next unless file.exist?
+
+        data = JSON.parse(file.read, symbolize_names: true)
+        context.send(:merge_step_data, step, data)
+      end
+
+      context
+    end
+
+    def initialize
+      @comments = []
+      @arguments = []
+      @relations = []
+      @cluster_results = {}
+      @umap_coords = nil
+      @initial_labels = {}
+      @labels = {}
+      @overview = nil
+      @result = nil
+      @output_dir = nil
+    end
+
+    # Save a step's output to file
+    #
+    # @param step_name [Symbol] The step name
+    # @param output_dir [String, Pathname] Output directory
+    def save_step(step_name, output_dir)
+      dir = Pathname.new(output_dir)
+      filename = OUTPUT_FILES[step_name]
+      return unless filename
+
+      FileUtils.mkdir_p(dir)
+      data = extract_step_output(step_name)
+      File.write(dir / filename, JSON.pretty_generate(data))
+    end
+
+    # Convert to hash for serialization
+    #
+    # @return [Hash]
+    def to_h
+      {
+        comments: @comments.map(&:to_h),
+        arguments: @arguments.map(&:to_h),
+        relations: @relations,
+        cluster_results: @cluster_results,
+        umap_coords: @umap_coords,
+        initial_labels: @initial_labels,
+        labels: @labels,
+        overview: @overview,
+        result: @result
+      }
+    end
+
+    private
+
+    def extract_step_output(step_name)
+      case step_name
+      when :extraction
+        {
+          comments: @comments.map(&:to_h),
+          arguments: @arguments.map(&:to_h),
+          relations: @relations
+        }
+      when :embedding
+        {
+          arguments: @arguments.map(&:to_embedding_h)
+        }
+      when :clustering
+        {
+          cluster_results: @cluster_results,
+          arguments: @arguments.map(&:to_clustering_h)
+        }
+      when :initial_labelling
+        { initial_labels: @initial_labels }
+      when :merge_labelling
+        { labels: @labels }
+      when :overview
+        { overview: @overview }
+      when :aggregation
+        @result
+      end
+    end
+
+    def merge_step_data(step_name, data)
+      case step_name
+      when :extraction
+        load_extraction_data(data)
+      when :embedding
+        merge_embedding_data(data)
+      when :clustering
+        merge_clustering_data(data)
+      when :initial_labelling
+        @initial_labels = data[:initial_labels] if data[:initial_labels]
+      when :merge_labelling
+        @labels = data[:labels] if data[:labels]
+      when :overview
+        @overview = data[:overview] if data[:overview]
+      end
+    end
+
+    def load_extraction_data(data)
+      @comments = (data[:comments] || []).map do |c|
+        Comment.new(**c.slice(:id, :body, :proposal_id, :source_url, :attributes, :properties))
+      end
+      @arguments = (data[:arguments] || []).map { |a| Argument.from_hash(a) }
+      @relations = data[:relations] if data[:relations]
+    end
+
+    def merge_embedding_data(data)
+      return unless data[:arguments]
+
+      embedding_map = data[:arguments].to_h { |e| [ e[:arg_id], e[:embedding] ] }
+      @arguments.each do |arg|
+        embedding = embedding_map[arg.arg_id]
+        arg.embedding = embedding if embedding
+      end
+    end
+
+    def merge_clustering_data(data)
+      @cluster_results = data[:cluster_results] if data[:cluster_results]
+      return unless data[:arguments]
+
+      clustering_map = data[:arguments].to_h { |c| [ c[:arg_id], c ] }
+      @arguments.each do |arg|
+        cluster_data = clustering_map[arg.arg_id]
+        next unless cluster_data
+
+        arg.x = cluster_data[:x]
+        arg.y = cluster_data[:y]
+        arg.cluster_ids = cluster_data[:cluster_ids]
+      end
+    end
+  end
+end

data/lib/broadlistening/csv_loader.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+require "csv"
+
+module Broadlistening
+  # Loads comments from CSV files with Kouchou-AI compatible format.
+  #
+  # This loader supports the CSV format used by Kouchou-AI (Python version),
+  # enabling compatibility testing between the two implementations.
+  #
+  # @example Loading a Kouchou-AI format CSV
+  #   comments = CsvLoader.load("inputs/example-polis.csv")
+  #
+  # @example Loading with property columns
+  #   comments = CsvLoader.load("data.csv", property_names: ["agrees", "disagrees"])
+  #
+  # @example Using custom column mapping
+  #   comments = CsvLoader.load("custom.csv", column_mapping: {
+  #     id: "my_id_column",
+  #     body: "my_body_column"
+  #   })
+  class CsvLoader
+    # Default column mapping for Kouchou-AI format
+    # Maps Ruby gem's expected keys to Kouchou-AI's CSV column names
+    KOUCHOU_AI_COLUMNS = {
+      id: "comment-id",
+      body: "comment-body",
+      source_url: "source-url"
+    }.freeze
+
+    class << self
+      # Load comments from a CSV file
+      #
+      # @param path [String] Path to the CSV file
+      # @param property_names [Array<String>] Property column names to extract
+      # @param column_mapping [Hash] Custom column name mapping (overrides defaults)
+      # @param encoding [String] File encoding (default: UTF-8 with BOM handling)
+      # @return [Array<Comment>] Array of Comment objects
+      def load(path, property_names: [], column_mapping: {}, encoding: "bom|utf-8")
+        mapping = KOUCHOU_AI_COLUMNS.merge(column_mapping)
+
+        comments = []
+        CSV.foreach(path, headers: true, encoding: encoding) do |row|
+          comment = build_comment(row, mapping, property_names)
+          comments << comment unless comment.nil?
+        end
+        comments
+      end
+
+      # Load comments from a CSV string
+      #
+      # @param csv_string [String] CSV content as string
+      # @param property_names [Array<String>] Property column names to extract
+      # @param column_mapping [Hash] Custom column name mapping
+      # @return [Array<Comment>] Array of Comment objects
+      def parse(csv_string, property_names: [], column_mapping: {})
+        mapping = KOUCHOU_AI_COLUMNS.merge(column_mapping)
+
+        comments = []
+        CSV.parse(csv_string, headers: true) do |row|
+          comment = build_comment(row, mapping, property_names)
+          comments << comment unless comment.nil?
+        end
+        comments
+      end
+
+      private
+
+      def build_comment(row, mapping, property_names)
+        id = extract_value(row, mapping[:id], "id")
+        body = extract_value(row, mapping[:body], "body")
+
+        # Skip rows without required fields
+        return nil if id.nil? || body.nil? || body.strip.empty?
+
+        hash = {
+          id: id.to_s,
+          body: body,
+          source_url: extract_value(row, mapping[:source_url], "source_url", "source-url")
+        }
+
+        # Extract attribute columns (attribute_* or attribute-*)
+        row.headers.each do |header|
+          next if header.nil?
+
+          if header.start_with?("attribute_") || header.start_with?("attribute-")
+            hash[header.to_sym] = row[header]
+          end
+        end
+
+        # Extract property columns
+        property_names.each do |prop_name|
+          value = row[prop_name] || row[prop_name.to_s.tr("_", "-")]
+          hash[prop_name.to_sym] = value if value
+        end
+
+        Comment.from_hash(hash, property_names: property_names)
+      end
+
+      def extract_value(row, *possible_names)
+        possible_names.compact.each do |name|
+          value = row[name]
+          return value unless value.nil?
+        end
+        nil
+      end
+    end
+  end
+end

data/lib/broadlistening/hierarchical_clustering.rb

@@ -0,0 +1,142 @@
+# frozen_string_literal: true
+
+module Broadlistening
+  class HierarchicalClustering
+    # Ward法による階層的クラスタリング
+    # scipy.cluster.hierarchy.linkage(method="ward") と同等の実装
+
+    class << self
+      def merge(centroids, labels, target_clusters)
+        new(centroids, labels, target_clusters).merge
+      end
+    end
+
+    def initialize(centroids, labels, target_clusters)
+      @centroids = to_numo_array(centroids)
+      @labels = labels.dup
+      @target_clusters = target_clusters
+      @n_original_clusters = @centroids.shape[0]
+    end
+
+    def merge
+      return @labels if current_cluster_count <= @target_clusters
+
+      # クラスタ情報を初期化
+      # 各クラスタ: {centroid: 重心, size: サイズ, members: 元のクラスタID}
+      clusters = initialize_clusters
+
+      # Ward法で階層的にマージ
+      while clusters.size > @target_clusters
+        c1_id, c2_id = find_ward_closest_pair(clusters)
+        break if c1_id.nil?
+
+        merge_ward_clusters!(clusters, c1_id, c2_id)
+      end
+
+      # 元のラベルを新しいクラスタIDにマッピング
+      build_final_labels(clusters)
+    end
+
+    private
+
+    def to_numo_array(centroids)
+      if centroids.is_a?(Numo::DFloat)
+        centroids
+      else
+        Numo::DFloat.cast(centroids)
+      end
+    end
+
+    def current_cluster_count
+      @labels.uniq.size
+    end
+
+    def initialize_clusters
+      clusters = {}
+      @n_original_clusters.times do |i|
+        clusters[i] = {
+          centroid: @centroids[i, true].to_a,
+          size: @labels.count(i),
+          members: [ i ]
+        }
+      end
+      clusters
+    end
+
+    def find_ward_closest_pair(clusters)
+      min_dist = Float::INFINITY
+      min_pair = [ nil, nil ]
+
+      cluster_ids = clusters.keys
+      cluster_ids.each_with_index do |c1_id, i|
+        cluster_ids[(i + 1)..].each do |c2_id|
+          dist = ward_distance(clusters[c1_id], clusters[c2_id])
+          if dist < min_dist
+            min_dist = dist
+            min_pair = [ c1_id, c2_id ]
+          end
+        end
+      end
+
+      min_pair
+    end
+
+    def ward_distance(cluster1, cluster2)
+      # Ward法: マージ時の分散増加量を計算
+      # d(i,j) = sqrt(2 * n_i * n_j / (n_i + n_j)) * ||c_i - c_j||
+      n1 = cluster1[:size]
+      n2 = cluster2[:size]
+      c1 = cluster1[:centroid]
+      c2 = cluster2[:centroid]
+
+      # ユークリッド距離の2乗
+      dist_sq = c1.zip(c2).sum { |a, b| (a - b)**2 }
+
+      # Ward距離
+      Math.sqrt(2.0 * n1 * n2 / (n1 + n2) * dist_sq)
+    end
+
+    def merge_ward_clusters!(clusters, c1_id, c2_id)
+      c1 = clusters[c1_id]
+      c2 = clusters[c2_id]
+
+      # 新しい重心を計算（サイズで重み付け）
+      n1 = c1[:size]
+      n2 = c2[:size]
+      new_size = n1 + n2
+
+      new_centroid = c1[:centroid].zip(c2[:centroid]).map do |v1, v2|
+        (v1 * n1 + v2 * n2) / new_size
+      end
+
+      # マージしたクラスタを作成（小さいIDを使用）
+      merged_id = [ c1_id, c2_id ].min
+      removed_id = [ c1_id, c2_id ].max
+
+      clusters[merged_id] = {
+        centroid: new_centroid,
+        size: new_size,
+        members: c1[:members] + c2[:members]
+      }
+
+      clusters.delete(removed_id)
+    end
+
+    def build_final_labels(clusters)
+      # 各元クラスタIDから最終クラスタIDへのマッピングを構築
+      original_to_final = {}
+      clusters.each_value do |cluster|
+        final_id = cluster[:members].min
+        cluster[:members].each do |original_id|
+          original_to_final[original_id] = final_id
+        end
+      end
+
+      # 連番に振り直し
+      unique_finals = original_to_final.values.uniq.sort
+      final_remap = unique_finals.each_with_index.to_h
+
+      @labels.map { |l| final_remap[original_to_final[l]] }
+    end
+  end
+end

data/lib/broadlistening/kmeans.rb

@@ -0,0 +1,185 @@
+# frozen_string_literal: true
+
+module Broadlistening
+  class KMeans
+    attr_reader :centroids, :labels, :n_clusters, :inertia
+
+    DEFAULT_MAX_ITERATIONS = 100
+    DEFAULT_TOLERANCE = 1e-6
+
+    def initialize(n_clusters:, max_iterations: DEFAULT_MAX_ITERATIONS, random_state: nil, tolerance: DEFAULT_TOLERANCE)
+      @n_clusters = n_clusters
+      @max_iterations = max_iterations
+      @tolerance = tolerance
+      @random = random_state ? Random.new(random_state) : Random.new
+      @centroids = nil
+      @labels = nil
+      @inertia = nil
+    end
+
+    def fit(data)
+      @data = to_numo_array(data)
+      validate_data!
+
+      @centroids = initialize_centroids_pp
+      @labels = Array.new(@data.shape[0])
+
+      @max_iterations.times do
+        @labels = assign_labels
+        new_centroids = update_centroids
+
+        if converged?(new_centroids)
+          @centroids = new_centroids
+          break
+        end
+
+        @centroids = new_centroids
+      end
+
+      @inertia = compute_inertia
+      self
+    end
+
+    def predict(data)
+      data = to_numo_array(data)
+      assign_labels_for(data)
+    end
+
+    def fit_predict(data)
+      fit(data)
+      @labels
+    end
+
+    private
+
+    def to_numo_array(data)
+      return data if data.is_a?(Numo::DFloat)
+
+      Numo::DFloat.cast(data)
+    end
+
+    def validate_data!
+      n_samples = @data.shape[0]
+      raise ClusteringError, "n_clusters (#{@n_clusters}) must be <= n_samples (#{n_samples})" if @n_clusters > n_samples
+      raise ClusteringError, "n_clusters must be positive" if @n_clusters <= 0
+    end
+
+    def initialize_centroids_pp
+      n_samples = @data.shape[0]
+      n_features = @data.shape[1]
+      centroids = Numo::DFloat.zeros(@n_clusters, n_features)
+
+      first_idx = @random.rand(n_samples)
+      centroids[0, true] = @data[first_idx, true]
+
+      (1...@n_clusters).each do |k|
+        distances = compute_min_distances_to_centroids(centroids[0...k, true])
+        probabilities = distances**2
+        sum_probs = probabilities.sum
+        probabilities /= sum_probs if sum_probs > 0
+
+        next_idx = weighted_random_choice(probabilities)
+        centroids[k, true] = @data[next_idx, true]
+      end
+
+      centroids
+    end
+
+    def compute_min_distances_to_centroids(centroids)
+      n_samples = @data.shape[0]
+      min_distances = Numo::DFloat.new(n_samples).fill(Float::INFINITY)
+
+      centroids.shape[0].times do |k|
+        distances = compute_distances_to_centroid(@data, centroids[k, true])
+        min_distances = Numo::DFloat.minimum(min_distances, distances)
+      end
+
+      min_distances
+    end
+
+    def compute_distances_to_centroid(points, centroid)
+      diff = points - centroid
+      (diff**2).sum(axis: 1)
+    end
+
+    def weighted_random_choice(probabilities)
+      cumsum = 0.0
+      threshold = @random.rand
+      probs_array = probabilities.to_a
+
+      probs_array.each_with_index do |prob, idx|
+        cumsum += prob
+        return idx if cumsum >= threshold
+      end
+
+      probs_array.size - 1
+    end
+
+    def assign_labels
+      assign_labels_for(@data)
+    end
+
+    def assign_labels_for(data)
+      n_samples = data.shape[0]
+      labels = Array.new(n_samples)
+
+      n_samples.times do |i|
+        point = data[i, true]
+        min_dist = Float::INFINITY
+        min_label = 0
+
+        @n_clusters.times do |k|
+          dist = squared_distance(point, @centroids[k, true])
+          if dist < min_dist
+            min_dist = dist
+            min_label = k
+          end
+        end
+
+        labels[i] = min_label
+      end
+
+      labels
+    end
+
+    def squared_distance(a, b)
+      ((a - b)**2).sum
+    end
+
+    def update_centroids
+      n_features = @data.shape[1]
+      new_centroids = Numo::DFloat.zeros(@n_clusters, n_features)
+      counts = Array.new(@n_clusters, 0)
+
+      @data.shape[0].times do |i|
+        label = @labels[i]
+        new_centroids[label, true] += @data[i, true]
+        counts[label] += 1
+      end
+
+      @n_clusters.times do |k|
+        if counts[k] > 0
+          new_centroids[k, true] /= counts[k]
+        else
+          random_idx = @random.rand(@data.shape[0])
+          new_centroids[k, true] = @data[random_idx, true]
+        end
+      end
+
+      new_centroids
+    end
+
+    def converged?(new_centroids)
+      ((@centroids - new_centroids)**2).sum < @tolerance
+    end
+
+    def compute_inertia
+      total = 0.0
+      @data.shape[0].times do |i|
+        label = @labels[i]
+        total += squared_distance(@data[i, true], @centroids[label, true])
+      end
+      total
+    end
+  end
+end

data/lib/broadlistening/llm_client.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+module Broadlistening
+  class LlmClient
+    MAX_RETRIES = 3
+    RETRY_DELAY = 1
+
+    def initialize(config)
+      @config = config
+      @provider = Provider.new(config.provider, local_llm_address: config.local_llm_address)
+      @client = @provider.build_openai_client(
+        api_key: config.api_key,
+        base_url: config.api_base_url,
+        azure_api_version: config.azure_api_version
+      )
+    end
+
+    def chat(system:, user:, json_mode: false)
+      params = build_chat_params(system, user, json_mode)
+      response = with_retry { @client.chat(parameters: params) }
+      extract_chat_content(response)
+    end
+
+    def embed(texts)
+      texts = [ texts ] if texts.is_a?(String)
+      response = with_retry do
+        @client.embeddings(
+          parameters: {
+            model: @config.embedding_model,
+            input: texts
+          }
+        )
+      end
+      extract_embeddings(response)
+    end
+
+    private
+
+    def build_chat_params(system, user, json_mode)
+      params = {
+        model: @config.model,
+        messages: [
+          { role: "system", content: system },
+          { role: "user", content: user }
+        ]
+      }
+      params[:response_format] = { type: "json_object" } if json_mode
+      params
+    end
+
+    def extract_chat_content(response)
+      validate_response!(response)
+      response.dig("choices", 0, "message", "content")
+    end
+
+    def extract_embeddings(response)
+      validate_response!(response)
+      response["data"].sort_by { |d| d["index"] }.map { |d| d["embedding"] }
+    end
+
+    def validate_response!(response)
+      return if response.is_a?(Hash) && !response["error"]
+
+      error_message = response.is_a?(Hash) ? response.dig("error", "message") : "Unknown error"
+      raise LlmError, "LLM API error: #{error_message}"
+    end
+
+    def with_retry
+      retries = 0
+      begin
+        yield
+      rescue Faraday::ClientError => e
+        raise LlmError, "LLM API error: #{e.message}"
+      rescue Faraday::ServerError, Faraday::ConnectionFailed, Faraday::TimeoutError,
+             Net::OpenTimeout, Errno::ECONNRESET => e
+        retries += 1
+        raise LlmError, "LLM API request failed after #{MAX_RETRIES} retries: #{e.message}" if retries > MAX_RETRIES
+
+        sleep(RETRY_DELAY * retries)
+        retry
+      end
+    end
+  end
+end