broadlistening 0.7.0

data/lib/broadlistening.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require "active_support"
+require "active_support/core_ext/string/inflections"
+require "numo/narray"
+require "openai"
+require "parallel"
+require "json"
+require "umappp"
+
+require_relative "broadlistening/version"
+require_relative "broadlistening/provider"
+require_relative "broadlistening/config"
+require_relative "broadlistening/spec_loader"
+require_relative "broadlistening/status"
+require_relative "broadlistening/planner"
+require_relative "broadlistening/comment"
+require_relative "broadlistening/argument"
+require_relative "broadlistening/csv_loader"
+require_relative "broadlistening/compatibility"
+require_relative "broadlistening/context"
+require_relative "broadlistening/pipeline"
+require_relative "broadlistening/cli"
+
+require_relative "broadlistening/llm_client"
+require_relative "broadlistening/kmeans"
+require_relative "broadlistening/hierarchical_clustering"
+
+# Steps
+require_relative "broadlistening/steps/base_step"
+require_relative "broadlistening/steps/extraction"
+require_relative "broadlistening/steps/embedding"
+require_relative "broadlistening/steps/clustering"
+require_relative "broadlistening/steps/initial_labelling"
+require_relative "broadlistening/steps/merge_labelling"
+require_relative "broadlistening/steps/overview"
+require_relative "broadlistening/steps/aggregation"
+
+module Broadlistening
+  class Error < StandardError; end
+  class ConfigurationError < Error; end
+  class LlmError < Error; end
+  class ClusteringError < Error; end
+end

data/schema/hierarchical_result.json

@@ -0,0 +1,152 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "https://github.com/takahashim/broadlistening-ruby/blob/main/schema/hierarchical_result.json",
+  "title": "Hierarchical Result",
+  "description": "Output schema for Kouchou-AI / Broadlistening pipeline results. This schema defines the structure of hierarchical_result.json produced by both Python and Ruby implementations.",
+  "type": "object",
+  "required": ["arguments", "clusters", "comments", "propertyMap", "translations", "overview", "config"],
+  "properties": {
+    "arguments": {
+      "type": "array",
+      "description": "List of extracted arguments/opinions from comments",
+      "items": {
+        "$ref": "#/definitions/argument"
+      }
+    },
+    "clusters": {
+      "type": "array",
+      "description": "Hierarchical cluster definitions",
+      "items": {
+        "$ref": "#/definitions/cluster"
+      }
+    },
+    "comments": {
+      "type": "object",
+      "description": "Map of comment_id to comment data",
+      "additionalProperties": {
+        "type": "object",
+        "properties": {
+          "comment": {
+            "type": "string",
+            "description": "Original comment text"
+          }
+        }
+      }
+    },
+    "propertyMap": {
+      "type": "object",
+      "description": "Map of property names to argument property values",
+      "additionalProperties": {
+        "type": "object",
+        "additionalProperties": true
+      }
+    },
+    "translations": {
+      "type": "object",
+      "description": "Translation data (reserved for future use)",
+      "additionalProperties": true
+    },
+    "overview": {
+      "type": "string",
+      "description": "LLM-generated overview summarizing all clusters"
+    },
+    "config": {
+      "type": "object",
+      "description": "Pipeline configuration used to generate this result",
+      "additionalProperties": true
+    },
+    "comment_num": {
+      "type": "integer",
+      "description": "Total number of input comments (optional)"
+    }
+  },
+  "definitions": {
+    "argument": {
+      "type": "object",
+      "description": "An extracted argument/opinion from a comment",
+      "required": ["arg_id", "argument", "comment_id", "x", "y", "cluster_ids"],
+      "properties": {
+        "arg_id": {
+          "type": "string",
+          "description": "Unique argument identifier in format A{comment_id}_{index}",
+          "pattern": "^A\\d+_\\d+$"
+        },
+        "argument": {
+          "type": "string",
+          "description": "The extracted argument/opinion text"
+        },
+        "comment_id": {
+          "type": ["integer", "string"],
+          "description": "ID of the source comment"
+        },
+        "x": {
+          "type": "number",
+          "description": "UMAP x-coordinate for visualization"
+        },
+        "y": {
+          "type": "number",
+          "description": "UMAP y-coordinate for visualization"
+        },
+        "p": {
+          "type": ["integer", "number"],
+          "description": "Priority or ordering value (optional)"
+        },
+        "cluster_ids": {
+          "type": "array",
+          "description": "List of cluster IDs this argument belongs to (from root to leaf)",
+          "items": {
+            "type": "string"
+          },
+          "minItems": 1
+        },
+        "attributes": {
+          "type": ["object", "null"],
+          "description": "Additional attributes extracted from source comment",
+          "additionalProperties": true
+        },
+        "url": {
+          "type": ["string", "null"],
+          "description": "Source URL of the original comment"
+        }
+      }
+    },
+    "cluster": {
+      "type": "object",
+      "description": "A cluster in the hierarchy",
+      "required": ["level", "id", "label", "takeaway", "value", "parent"],
+      "properties": {
+        "level": {
+          "type": "integer",
+          "description": "Hierarchy level (0 = root)",
+          "minimum": 0
+        },
+        "id": {
+          "type": "string",
+          "description": "Unique cluster identifier. Root is '0', others are '{level}_{index}'",
+          "pattern": "^(0|\\d+_\\d+)$"
+        },
+        "label": {
+          "type": "string",
+          "description": "LLM-generated label for this cluster"
+        },
+        "takeaway": {
+          "type": "string",
+          "description": "LLM-generated description/takeaway for this cluster"
+        },
+        "value": {
+          "type": "integer",
+          "description": "Number of arguments in this cluster",
+          "minimum": 0
+        },
+        "parent": {
+          "type": "string",
+          "description": "ID of parent cluster (empty string for root)"
+        },
+        "density_rank_percentile": {
+          "type": ["number", "null"],
+          "description": "Density ranking percentile (optional)"
+        }
+      }
+    }
+  }
+}

data/sig/broadlistening.rbs

@@ -0,0 +1,4 @@
+module Broadlistening
+  VERSION: String
+  # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+end

metadata

@@ -0,0 +1,194 @@
+--- !ruby/object:Gem::Specification
+name: broadlistening
+version: !ruby/object:Gem::Version
+  version: 0.7.0
+platform: ruby
+authors:
+- takahashim
+bindir: exe
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '7.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '7.0'
+- !ruby/object:Gem::Dependency
+  name: csv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: numo-narray
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+- !ruby/object:Gem::Dependency
+  name: parallel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.20'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.20'
+- !ruby/object:Gem::Dependency
+  name: rice
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.6.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.6.0
+- !ruby/object:Gem::Dependency
+  name: ruby-openai
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '7.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '7.0'
+- !ruby/object:Gem::Dependency
+  name: umappp
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+- !ruby/object:Gem::Dependency
+  name: json_schemer
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+description: A Ruby implementation of the Broadlistening pipeline for clustering and
+  analyzing public comments using LLM
+email:
+- takahashimm@gmail.com
+executables:
+- broadlistening
+extensions: []
+extra_rdoc_files: []
+files:
+- ".rspec"
+- ".rubocop.yml"
+- CHANGELOG.md
+- CLAUDE.md
+- LICENSE
+- LICENSE-AGPLv3.txt
+- README.md
+- Rakefile
+- exe/broadlistening
+- lib/broadlistening.rb
+- lib/broadlistening/argument.rb
+- lib/broadlistening/cli.rb
+- lib/broadlistening/comment.rb
+- lib/broadlistening/compatibility.rb
+- lib/broadlistening/config.rb
+- lib/broadlistening/context.rb
+- lib/broadlistening/csv_loader.rb
+- lib/broadlistening/hierarchical_clustering.rb
+- lib/broadlistening/kmeans.rb
+- lib/broadlistening/llm_client.rb
+- lib/broadlistening/pipeline.rb
+- lib/broadlistening/planner.rb
+- lib/broadlistening/provider.rb
+- lib/broadlistening/spec_loader.rb
+- lib/broadlistening/status.rb
+- lib/broadlistening/steps/aggregation.rb
+- lib/broadlistening/steps/base_step.rb
+- lib/broadlistening/steps/clustering.rb
+- lib/broadlistening/steps/embedding.rb
+- lib/broadlistening/steps/extraction.rb
+- lib/broadlistening/steps/initial_labelling.rb
+- lib/broadlistening/steps/merge_labelling.rb
+- lib/broadlistening/steps/overview.rb
+- lib/broadlistening/version.rb
+- schema/hierarchical_result.json
+- sig/broadlistening.rbs
+homepage: https://github.com/takahashim/broadlistening-ruby
+licenses:
+- AGPL-3.0
+metadata:
+  homepage_uri: https://github.com/takahashim/broadlistening-ruby
+  source_code_uri: https://github.com/takahashim/broadlistening-ruby
+  changelog_uri: https://github.com/takahashim/broadlistening-ruby/blob/main/CHANGELOG.md
+  rubygems_mfa_required: 'true'
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.1.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.7
+specification_version: 4
+summary: Broadlistening pipeline for opinion analysis
+test_files: []