broadlistening 0.7.0

data/lib/broadlistening/steps/aggregation.rb

@@ -0,0 +1,228 @@
+# frozen_string_literal: true
+
+require "csv"
+
+module Broadlistening
+  module Steps
+    class Aggregation < BaseStep
+      CSV_FILENAME = "final_result_with_comments.csv"
+
+      # Output format compatible with Kouchou-AI Python implementation
+      def execute
+        result = {
+          arguments: build_arguments,
+          clusters: build_clusters,
+          comments: build_comments,
+          propertyMap: build_property_map,
+          translations: build_translations,
+          overview: context.overview,
+          config: config.to_h,
+          comment_num: context.comments.size
+        }
+
+        context.result = result
+
+        export_csv if config.is_pubcom && context.output_dir
+
+        context
+      end
+
+      private
+
+      def build_arguments
+        context.arguments.map do |arg|
+          build_single_argument(arg)
+        end
+      end
+
+      def build_single_argument(arg)
+        result = {
+          arg_id: arg.arg_id,
+          argument: arg.argument,
+          comment_id: arg.comment_id_int,
+          x: arg.x&.to_f,
+          y: arg.y&.to_f,
+          p: 0,
+          cluster_ids: arg.cluster_ids
+        }
+
+        result[:attributes] = arg.attributes if arg.attributes
+        result[:url] = arg.url if config.enable_source_link && arg.url
+
+        result
+      end
+
+      def build_clusters
+        clusters = [ root_cluster ]
+
+        context.labels.each_value do |label|
+          clusters << {
+            level: label[:level],
+            id: label[:cluster_id],
+            label: label[:label],
+            takeaway: label[:description] || "",
+            value: count_arguments_in_cluster(label[:cluster_id]),
+            parent: find_parent_cluster(label),
+            density_rank_percentile: nil
+          }
+        end
+
+        clusters.sort_by { |c| [ c[:level], c[:id] ] }
+      end
+
+      def root_cluster
+        {
+          level: 0,
+          id: "0",
+          label: "全体",
+          takeaway: "",
+          value: context.arguments.size,
+          parent: "",
+          density_rank_percentile: nil
+        }
+      end
+
+      def count_arguments_in_cluster(cluster_id)
+        context.arguments.count { |arg| arg.in_cluster?(cluster_id) }
+      end
+
+      def find_parent_cluster(label)
+        return "0" if label[:level] == 1
+
+        parent_level = label[:level] - 1
+
+        # Find an argument that belongs to this cluster
+        arg_idx = context.arguments.index { |arg| arg.in_cluster?(label[:cluster_id]) }
+        return "0" unless arg_idx
+
+        parent_cluster_num = context.cluster_results[parent_level][arg_idx]
+        "#{parent_level}_#{parent_cluster_num}"
+      end
+
+      def build_comments
+        comments_with_args = Set.new
+        context.arguments.each do |arg|
+          comments_with_args.add(arg.comment_id_int)
+        end
+
+        result = {}
+        context.comments.each do |comment|
+          comment_id = comment.id.to_i
+          next unless comments_with_args.include?(comment_id)
+
+          result[comment_id.to_s] = { comment: comment.body }
+        end
+
+        result
+      end
+
+      def build_property_map
+        return {} if config.property_names.empty?
+
+        property_map = {}
+        config.property_names.each do |prop_name|
+          property_map[prop_name.to_s] = {}
+        end
+
+        context.arguments.each do |arg|
+          next unless arg.properties
+
+          arg.properties.each do |prop_name, value|
+            property_map[prop_name.to_s] ||= {}
+            property_map[prop_name.to_s][arg.arg_id] = normalize_property_value(value)
+          end
+        end
+
+        property_map
+      end
+
+      def normalize_property_value(value)
+        return nil if value.nil?
+
+        case value
+        when Integer, Float, String, TrueClass, FalseClass
+          value
+        when Array
+          value.map { |v| normalize_property_value(v) }
+        else
+          value.to_s
+        end
+      end
+
+      def build_translations
+        {}
+      end
+
+      # Export CSV with original comments for pubcom mode
+      def export_csv
+        csv_path = Pathname.new(context.output_dir) / CSV_FILENAME
+        level1_labels = build_level1_label_map
+
+        CSV.open(csv_path, "w", encoding: "UTF-8") do |csv|
+          csv << csv_headers
+          context.arguments.each do |arg|
+            csv << build_csv_row(arg, level1_labels)
+          end
+        end
+      end
+
+      def csv_headers
+        headers = %w[comment_id original_comment arg_id argument category_id category x y]
+        headers += attribute_columns
+        headers
+      end
+
+      def build_csv_row(arg, level1_labels)
+        comment = find_comment(arg.comment_id)
+        level1_cluster_id = find_level1_cluster_id(arg)
+        category_label = level1_labels[level1_cluster_id] || ""
+
+        row = [
+          arg.comment_id,
+          comment&.body || "",
+          arg.arg_id,
+          arg.argument,
+          level1_cluster_id,
+          category_label,
+          arg.x,
+          arg.y
+        ]
+
+        # Add attribute values
+        attribute_columns.each do |attr_name|
+          row << (arg.attributes&.dig(attr_name.sub(/^attribute_/, "")) || comment&.attributes&.dig(attr_name.sub(/^attribute_/, "")))
+        end
+
+        row
+      end
+
+      def build_level1_label_map
+        context.labels
+          .select { |_, label| label[:level] == 1 }
+          .transform_values { |label| label[:label] }
+          .transform_keys(&:to_s)
+      end
+
+      def find_level1_cluster_id(arg)
+        arg.cluster_ids&.find { |id| id.start_with?("1_") } || ""
+      end
+
+      def find_comment(comment_id)
+        context.comments.find { |c| c.id.to_s == comment_id.to_s }
+      end
+
+      def attribute_columns
+        @attribute_columns ||= begin
+          attrs = Set.new
+          context.arguments.each do |arg|
+            arg.attributes&.each_key { |k| attrs.add("attribute_#{k}") }
+          end
+          context.comments.each do |comment|
+            comment.attributes&.each_key { |k| attrs.add("attribute_#{k}") }
+          end
+          attrs.to_a.sort
+        end
+      end
+    end
+  end
+end

data/lib/broadlistening/steps/base_step.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Broadlistening
+  module Steps
+    class BaseStep
+      attr_reader :config, :context
+
+      # @param config [Config] Pipeline configuration
+      # @param context [Context] Pipeline context
+      def initialize(config, context)
+        @config = config
+        @context = context
+
+        raise ArgumentError, "context must be a Context, got #{context.class}" unless context.is_a?(Context)
+      end
+
+      def execute
+        raise NotImplementedError, "#{self.class} must implement #execute"
+      end
+
+      protected
+
+      def llm_client
+        @llm_client ||= LlmClient.new(config)
+      end
+
+      def instrument(event_name, payload = {}, &block)
+        ActiveSupport::Notifications.instrument(event_name, payload, &block)
+      end
+
+      def notify_progress(current:, total:, message: nil)
+        instrument("progress.broadlistening", {
+          step: self.class.name.demodulize.underscore,
+          current: current,
+          total: total,
+          percentage: total.positive? ? (current.to_f / total * 100).round(1) : 0,
+          message: message
+        })
+      end
+    end
+  end
+end

data/lib/broadlistening/steps/clustering.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+module Broadlistening
+  module Steps
+    class Clustering < BaseStep
+      def execute
+        return context if context.arguments.empty?
+
+        embeddings = build_embeddings_matrix(context.arguments)
+        umap_coords = perform_umap(embeddings)
+        cluster_results = perform_hierarchical_clustering(umap_coords)
+
+        assign_cluster_info_to_arguments(context.arguments, umap_coords, cluster_results)
+
+        context.cluster_results = cluster_results
+        context.umap_coords = umap_coords
+        context
+      end
+
+      private
+
+      def build_embeddings_matrix(arguments)
+        Numo::DFloat.cast(arguments.map(&:embedding))
+      end
+
+      def perform_umap(embeddings)
+        n_samples = embeddings.shape[0]
+        num_neighbors = [ 15, n_samples - 1 ].min
+
+        # Convert to SFloat for umappp (required format)
+        embeddings_sfloat = Numo::SFloat.cast(embeddings)
+
+        # Umappp.run returns 2D coordinates
+        result = Umappp.run(
+          embeddings_sfloat,
+          ndim: 2,
+          num_neighbors: num_neighbors,
+          seed: 42
+        )
+
+        # Convert back to DFloat for consistency
+        Numo::DFloat.cast(result)
+      end
+
+      def perform_hierarchical_clustering(umap_coords)
+        cluster_nums = config.cluster_nums.sort
+        n_samples = umap_coords.shape[0]
+
+        # Adjust cluster numbers if we have fewer samples
+        adjusted_cluster_nums = cluster_nums.map { |n| [ n, n_samples ].min }.uniq
+
+        max_clusters = adjusted_cluster_nums.last
+
+        # Perform KMeans with max clusters
+        kmeans = KMeans.new(
+          n_clusters: max_clusters,
+          random_state: 42
+        )
+        kmeans.fit(umap_coords)
+
+        # Build hierarchical results
+        build_hierarchical_results(kmeans, adjusted_cluster_nums)
+      end
+
+      def build_hierarchical_results(kmeans, cluster_nums)
+        results = {}
+
+        cluster_nums[0..-2].each_with_index do |n_target, level|
+          merged_labels = HierarchicalClustering.merge(
+            kmeans.centroids,
+            kmeans.labels,
+            n_target
+          )
+          results[level + 1] = merged_labels
+        end
+
+        # Final level uses KMeans labels directly
+        results[cluster_nums.size] = kmeans.labels
+
+        results
+      end
+
+      def assign_cluster_info_to_arguments(arguments, umap_coords, cluster_results)
+        arguments.each_with_index do |arg, idx|
+          arg.x = umap_coords[idx, 0]
+          arg.y = umap_coords[idx, 1]
+          arg.cluster_ids = build_cluster_ids(idx, cluster_results)
+        end
+      end
+
+      def build_cluster_ids(idx, cluster_results)
+        cluster_ids = [ "0" ] # Root cluster
+
+        cluster_results.keys.sort.each do |level|
+          cluster_id = "#{level}_#{cluster_results[level][idx]}"
+          cluster_ids << cluster_id
+        end
+
+        cluster_ids
+      end
+    end
+  end
+end

data/lib/broadlistening/steps/embedding.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Broadlistening
+  module Steps
+    class Embedding < BaseStep
+      BATCH_SIZE = 1000
+
+      def execute
+        return context if context.arguments.empty?
+
+        embeddings = compute_embeddings(context.arguments)
+        attach_embeddings_to_arguments(context.arguments, embeddings)
+
+        context
+      end
+
+      private
+
+      def compute_embeddings(arguments)
+        texts = arguments.map(&:argument)
+        embeddings = []
+        total_batches = (texts.size.to_f / BATCH_SIZE).ceil
+
+        texts.each_slice(BATCH_SIZE).with_index(1) do |batch, batch_num|
+          batch_embeddings = llm_client.embed(batch)
+          embeddings.concat(batch_embeddings)
+          notify_progress(current: batch_num, total: total_batches)
+        end
+
+        embeddings
+      end
+
+      def attach_embeddings_to_arguments(arguments, embeddings)
+        arguments.each_with_index do |arg, idx|
+          arg.embedding = embeddings[idx]
+        end
+      end
+    end
+  end
+end

data/lib/broadlistening/steps/extraction.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module Broadlistening
+  module Steps
+    class Extraction < BaseStep
+      def execute
+        return context if context.comments.empty?
+
+        results = extract_opinions_in_parallel(context.comments)
+        build_arguments_and_relations(context.comments, results)
+
+        context
+      end
+
+      private
+
+      def extract_opinions_in_parallel(comments)
+        total = comments.size
+        mutex = Mutex.new
+        processed = 0
+
+        Parallel.map(comments, in_threads: config.workers) do |comment|
+          result = extract_arguments_from_comment(comment)
+          current = mutex.synchronize { processed += 1 }
+          notify_progress(current: current, total: total)
+          result
+        end
+      end
+
+      def extract_arguments_from_comment(comment)
+        return [] if comment.empty?
+
+        response = llm_client.chat(
+          system: config.prompts[:extraction],
+          user: comment.body,
+          json_mode: true
+        )
+        parse_extraction_response(response)
+      rescue StandardError => e
+        warn "Failed to extract from comment #{comment.id}: #{e.message}"
+        []
+      end
+
+      def parse_extraction_response(response)
+        parsed = JSON.parse(response)
+        opinions = parsed["extractedOpinionList"] || parsed["opinions"] || []
+        opinions.select { |o| o.is_a?(String) && !o.strip.empty? }
+      rescue JSON::ParserError
+        parse_fallback_response(response)
+      end
+
+      def parse_fallback_response(response)
+        response.split("\n").map(&:strip).reject(&:empty?)
+      end
+
+      def build_arguments_and_relations(comments, results)
+        results.each_with_index do |extracted_opinions, idx|
+          comment = comments[idx]
+          extracted_opinions.each_with_index do |opinion_text, opinion_idx|
+            arg = Argument.from_comment(comment, opinion_text, opinion_idx)
+            context.arguments << arg
+
+            context.relations << {
+              arg_id: arg.arg_id,
+              comment_id: arg.comment_id,
+              proposal_id: comment.proposal_id
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/broadlistening/steps/initial_labelling.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+module Broadlistening
+  module Steps
+    class InitialLabelling < BaseStep
+      SAMPLING_NUM = 30
+
+      def execute
+        return context if context.arguments.empty? || context.cluster_results.empty?
+
+        max_level = context.cluster_results.keys.max
+        cluster_ids = context.cluster_results[max_level].uniq
+
+        labels = label_clusters_in_parallel(context.arguments, max_level, cluster_ids)
+
+        context.initial_labels = labels.to_h { |l| [ l[:cluster_id], l ] }
+        context
+      end
+
+      private
+
+      def label_clusters_in_parallel(arguments, level, cluster_ids)
+        total = cluster_ids.size
+        mutex = Mutex.new
+        processed = 0
+
+        Parallel.map(cluster_ids, in_threads: config.workers) do |cluster_id|
+          result = label_single_cluster(arguments, level, cluster_id)
+          current = mutex.synchronize { processed += 1 }
+          notify_progress(current: current, total: total)
+          result
+        end
+      end
+
+      def label_single_cluster(arguments, level, cluster_id)
+        cluster_args = filter_arguments_by_cluster(arguments, level, cluster_id)
+        sampled = sample_arguments(cluster_args)
+
+        input = sampled.map(&:argument).join("\n")
+
+        response = llm_client.chat(
+          system: config.prompts[:initial_labelling],
+          user: input,
+          json_mode: true
+        )
+
+        parse_label_response(response, level, cluster_id)
+      rescue StandardError => e
+        warn "Failed to label cluster #{level}_#{cluster_id}: #{e.message}"
+        default_label(level, cluster_id)
+      end
+
+      def filter_arguments_by_cluster(arguments, level, cluster_id)
+        target_cluster_id = "#{level}_#{cluster_id}"
+        arguments.select { |arg| arg.in_cluster?(target_cluster_id) }
+      end
+
+      def sample_arguments(cluster_args)
+        sample_size = [ SAMPLING_NUM, cluster_args.size ].min
+        cluster_args.sample(sample_size)
+      end
+
+      def parse_label_response(response, level, cluster_id)
+        parsed = JSON.parse(response)
+        {
+          cluster_id: "#{level}_#{cluster_id}",
+          level: level,
+          label: parsed["label"] || "グループ#{cluster_id}",
+          description: parsed["description"] || ""
+        }
+      rescue JSON::ParserError
+        default_label(level, cluster_id)
+      end
+
+      def default_label(level, cluster_id)
+        {
+          cluster_id: "#{level}_#{cluster_id}",
+          level: level,
+          label: "グループ#{cluster_id}",
+          description: ""
+        }
+      end
+    end
+  end
+end

data/lib/broadlistening/steps/merge_labelling.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+module Broadlistening
+  module Steps
+    class MergeLabelling < BaseStep
+      def execute
+        return context if context.initial_labels.empty?
+
+        all_labels = context.initial_labels.dup
+
+        # Build parent-child relationships and merge from bottom to top
+        levels = context.cluster_results.keys.sort.reverse
+        levels[1..].each do |level|
+          parent_labels = merge_labels_for_level(context.arguments, all_labels, context.cluster_results, level)
+          parent_labels.each { |l| all_labels[l[:cluster_id]] = l }
+        end
+
+        context.labels = all_labels
+        context
+      end
+
+      private
+
+      def merge_labels_for_level(arguments, all_labels, cluster_results, level)
+        child_level = level + 1
+        parent_clusters = cluster_results[level].uniq
+        total = parent_clusters.size
+        mutex = Mutex.new
+        processed = 0
+
+        Parallel.map(parent_clusters, in_threads: config.workers) do |parent_cluster_id|
+          result = merge_single_parent(arguments, all_labels, cluster_results, level, child_level, parent_cluster_id)
+          current = mutex.synchronize { processed += 1 }
+          notify_progress(current: current, total: total, message: "level #{level}")
+          result
+        end
+      end
+
+      def merge_single_parent(arguments, all_labels, cluster_results, level, child_level, parent_cluster_id)
+        child_cluster_ids = find_child_clusters(arguments, cluster_results, level, child_level, parent_cluster_id)
+        child_labels = child_cluster_ids.filter_map { |cid| all_labels["#{child_level}_#{cid}"] }
+
+        return default_label(level, parent_cluster_id) if child_labels.empty?
+
+        input = child_labels.map { |l| "- #{l[:label]}: #{l[:description]}" }.join("\n")
+
+        response = llm_client.chat(
+          system: config.prompts[:merge_labelling],
+          user: input,
+          json_mode: true
+        )
+
+        parse_label_response(response, level, parent_cluster_id)
+      rescue StandardError => e
+        warn "Failed to merge labels for cluster #{level}_#{parent_cluster_id}: #{e.message}"
+        default_label(level, parent_cluster_id)
+      end
+
+      def find_child_clusters(arguments, cluster_results, parent_level, child_level, parent_cluster_id)
+        child_clusters = Set.new
+
+        arguments.each_with_index do |_arg, idx|
+          next unless cluster_results[parent_level][idx] == parent_cluster_id
+
+          child_clusters.add(cluster_results[child_level][idx])
+        end
+
+        child_clusters.to_a
+      end
+
+      def parse_label_response(response, level, cluster_id)
+        parsed = JSON.parse(response)
+        {
+          cluster_id: "#{level}_#{cluster_id}",
+          level: level,
+          label: parsed["label"] || "グループ#{cluster_id}",
+          description: parsed["description"] || ""
+        }
+      rescue JSON::ParserError
+        default_label(level, cluster_id)
+      end
+
+      def default_label(level, cluster_id)
+        {
+          cluster_id: "#{level}_#{cluster_id}",
+          level: level,
+          label: "グループ#{cluster_id}",
+          description: ""
+        }
+      end
+    end
+  end
+end

data/lib/broadlistening/steps/overview.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Broadlistening
+  module Steps
+    class Overview < BaseStep
+      def execute
+        return context if context.labels.empty?
+
+        top_labels = find_top_level_labels(context.labels)
+        overview = generate_overview(top_labels)
+
+        context.overview = overview
+        context
+      end
+
+      private
+
+      def find_top_level_labels(labels)
+        min_level = labels.values.map { |l| l[:level] }.min
+        labels.values.select { |l| l[:level] == min_level }
+      end
+
+      def generate_overview(top_labels)
+        input = top_labels.map { |l| "- #{l[:label]}: #{l[:description]}" }.join("\n")
+
+        llm_client.chat(
+          system: config.prompts[:overview],
+          user: input
+        )
+      rescue StandardError => e
+        warn "Failed to generate overview: #{e.message}"
+        ""
+      end
+    end
+  end
+end

data/lib/broadlistening/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Broadlistening
+  VERSION = "0.7.0"
+end