gtfs_stops_clustering 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 6a9a16f2fa8980f4bd7e84e9912e7925caa556fd2b2daab56209ff8351e93a19
+  data.tar.gz: 652b0895096c0b55009d669ea1b8416368dec5884b4a395ca4fd46c762836575
+SHA512:
+  metadata.gz: 3ea8cd7f921ff06aa5e838684d1efb8fcaa48fa3b2601e85ed6f926fecebc9d4bb8831399d5bd0e63cf1a2531231e5d8c71b4e56a3287c9ef0aa602b22726a7f
+  data.tar.gz: 87bea9a4711300f07b9b6a0151590de265e6c55447a9c1e6ad3b7503b1ffaa45ac5090e1b1859a4786765b1709826519f84876504faac815d7c85282ee1857ed

data/lib/gtfs_stops_clustering/data_import.rb

@@ -0,0 +1,60 @@
+# lib/data_import.rb
+
+require 'csv'
+require 'gtfs'
+
+module DataImport
+  VERSION='0.0.1'
+  attr_accessor :data_import
+
+  class DataImport
+    attr_accessor :stops, :stops_config_file, :stops_names, :stops_corner_cases, :stops_data, :stops_redis_geodata
+
+    def initialize(stops, stops_config_file)
+      @stops = stops
+      @stops_config_file = stops_config_file
+      @stops_corner_cases = []
+      @stops_names = []
+      @stops_data = []
+      @stops_redis_geodata = []
+      import_stops_corner_cases
+      import_stops_data
+    end
+
+    def import_stops_corner_cases
+      if File.exist?(@stops_config_file)
+        CSV.foreach(@stops_config_file, headers: true) do |row|
+          stop_name = row['stop_name']
+          cluster_name = row['cluster_name']
+
+          stops_corner_cases << { stop_name: stop_name, cluster_name: cluster_name }
+        end
+      end
+    end
+
+    def import_stops_data
+      @stops.each do |row|
+        latitude = row.lat
+        longitude = row.lon
+        stop_name = row.name
+
+        stop_name = @stops_corner_cases.find { |entry| entry[:stop_name] == stop_name }[:cluster_name] if stops_corner_cases.find { |entry| entry[:stop_name] == stop_name }
+
+        @stops_names << stop_name
+        @stops_data << [latitude, longitude]
+        @stops_redis_geodata << [longitude, latitude, "#{longitude},#{latitude}"]
+      end
+    end
+  end
+
+  def import_stops_data(*args)
+    @data_import = DataImport.new(*args)
+    {
+      stops_data: @data_import.stops_data,
+      stops_names: @data_import.stops_names,
+      stops_redis_geodata: @data_import.stops_redis_geodata
+    }
+  end
+end
+
+include DataImport

data/lib/gtfs_stops_clustering/dbscan.rb

@@ -0,0 +1,180 @@
+## https://github.com/shiguodong/dbscan (fork)
+
+require 'distance_measures'
+require 'text'
+require 'geocoder'
+require_relative 'redis_geodata'
+
+class Array
+  def haversine_distance2(n)
+    Geocoder::Calculations.distance_between(self, n)
+  end
+end
+
+module DBSCAN
+  class Clusterer
+    attr_accessor :points, :options, :clusters
+
+    def initialize(points, stops_redis_geodata, options = {})
+      options[:distance] = :euclidean_distance unless options[:distance]
+      options[:labels] = [] unless options[:labels]
+
+      c = 0
+      redis_geodata_import(stops_redis_geodata, options[:epsilon])
+      @points = points.map { |e| po = Point.new(e, options[:labels][c]); c +=1; po }
+      @options = options
+      @clusters = {-1 => []}
+
+      clusterize!
+    end
+
+    def clusterize!
+      current_cluster = -1
+      @points.each do |point|
+        next if point.visited?
+
+        point.visit!
+        neighbors = inmediate_neighbors(point)
+
+        if neighbors.size >= options[:min_points]
+          current_cluster += 1
+          point.cluster = current_cluster
+          cluster = [point].push(add_connected(neighbors, current_cluster))
+          clusters[current_cluster] = cluster.flatten
+
+          # Get Cluster Name
+          labels = clusters[current_cluster].map { |e| e.label.capitalize }
+          cluster_name = find_cluster_name(labels)
+
+          # Get Cluster Position
+          cluster_pos = find_cluster_position(clusters[current_cluster])
+
+          clusters[current_cluster].each { |e|
+            e.cluster_name = cluster_name
+            e.cluster_pos = cluster_pos
+          }
+        else
+          clusters[-1].push(point)
+        end
+      end
+    end
+
+    def results
+      hash = {}
+      @clusters.dup.each { |cluster_index, value| hash[cluster_index] = value.flatten.map(&:items) unless value.flatten.empty? }
+      hash
+    end
+
+    def labeled_results
+      hash = {}
+      @clusters.each do |cluster_index, elements|
+        hash.store(cluster_index, [])
+        elements.each do |e|
+          hash[cluster_index].push(
+            {
+              stop_id: nil,
+              stop_code: nil,
+              cluster_name: e.cluster_name,
+              cluster_pos: e.cluster_pos,
+              stop_name: e.label,
+              stop_lat: e.items[0],
+              stop_lon: e.items[1],
+              parent_station: nil
+            }
+          )
+        end
+      end
+      hash
+    end
+
+    def inmediate_neighbors(point)
+      neighbors = []
+      geosearch_results = geosearch(point.items[1], point.items[0])
+      geosearch_results.each do |neighbor_pos|
+        coordinates = neighbor_pos.split(',')
+        neighbor = @points.find { |point| point.items[0] == coordinates[1] &&
+                                  point.items[1] == coordinates[0] }
+        next unless neighbor
+
+        string_distance = Text::Levenshtein.distance(point.label.downcase, neighbor.label.downcase)
+        similarity = 1 - string_distance.to_f / [point.label.length, point.label.length].max
+        neighbors.push(neighbor) if similarity > options[:similarity]
+      end
+      neighbors
+    end
+
+    def add_connected(neighbors, current_cluster)
+      cluster_points = []
+      neighbors.each do |point|
+        unless point.visited?
+          point.visit!
+          new_points = inmediate_neighbors(point)
+
+          if new_points.size >= options[:min_points]
+            new_points.each do |p|
+              unless neighbors.include?(p)
+                neighbors.push(p)
+              end
+            end
+          end
+        end
+
+        unless point.cluster
+          cluster_points.push(point)
+          point.cluster = current_cluster
+        end
+      end
+
+      cluster_points
+    end
+  end
+
+  def find_cluster_name(labels)
+    words = labels.map { |label| label.strip.split }
+    common_title = ''
+
+    # Loop through each word index starting from the first
+    (0...words.first.length).each do |i|
+      words_at_index = words.map { |word_list| word_list[i] }
+
+      break unless words_at_index.uniq.length == 1
+
+      common_title += " #{words_at_index.first.capitalize}"
+    end
+
+    common_title.strip! ? common_title : labels.first
+  end
+
+  def find_cluster_position(cluster)
+    total_lat = cluster.map { |e| e.items[0].to_f }.sum
+    total_lon = cluster.map { |e| e.items[1].to_f }.sum
+    avg_lat = total_lat / cluster.size
+    avg_lon = total_lon / cluster.size
+    [avg_lat, avg_lon]
+  end
+
+  class Point
+    attr_accessor :items, :cluster, :visited, :label, :cluster_name, :cluster_pos
+
+    define_method(:visited?) { @visited }
+    define_method(:visit!) { @visited = true }
+    def initialize(point, label)
+      @items,
+      @cluster,
+      @visited,
+      @label = point,
+      nil,
+      false,
+      label,
+      @cluster_name,
+      @cluster_pos = []
+    end
+  end
+
+  def DBSCAN(* args)
+    clusterer = Clusterer.new(*args)
+    clusterer.labeled_results
+  end
+end
+
+include DBSCAN

data/lib/gtfs_stops_clustering/redis_geodata.rb

@@ -0,0 +1,40 @@
+# lib/redis_geodata.rb
+require 'redis'
+
+module RedisGeodata
+  VERSION='0.0.1'
+  attr_accessor :redis
+
+  class RedisGeodata
+    attr_accessor :stops, :key, :redis, :epsilon
+
+    def initialize(stops, epsilon)
+      @redis = Redis.new(url: 'redis://127.0.0.1:6379')
+      @stops = stops
+      @key = 'stops'
+      @epsilon = epsilon
+      geoadd
+    end
+
+    def geoadd
+      @redis.geoadd(@key, *@stops)
+      @redis.expire(@key, 100_000_0)
+    end
+
+    def geosearch(longitude, latitude)
+      list = @redis.georadius(@key, longitude, latitude, @epsilon, 'km')
+      list.reject! { |point| point == longitude.to_s + "," + latitude.to_s }
+      list
+    end
+  end
+
+  def redis_geodata_import(*args)
+    @redis = RedisGeodata.new(*args)
+  end
+
+  def geosearch(*args)
+    @redis.geosearch(*args)
+  end
+end
+
+include RedisGeodata

data/lib/gtfs_stops_clustering/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module GtfsStopsClustering
+  VERSION = "0.1.0"
+end

data/lib/gtfs_stops_clustering.rb

@@ -0,0 +1,69 @@
+# lib/gtfs_stops_clustering.rb
+
+require 'gtfs'
+require 'csv'
+require_relative './gtfs_stops_clustering/data_import'
+require_relative './gtfs_stops_clustering/dbscan'
+
+module GtfsStopsClustering
+  VERSION='0.0.1'
+  attr_accessor :gtfs_stops_clustering
+
+  class GtfsStopsClustering
+    attr_accessor :clusters, :gtfs_urls, :gtfs_stops, :stops_config_path, :epsilon, :min_points, :names_similarity
+
+    def initialize(gtfs_urls, epsilon, min_points, names_similarity, stops_config_path)
+      @clusters = []
+      unless gtfs_urls.empty?
+        @gtfs_paths = gtfs_urls
+        @stops_config_path = stops_config_path
+        @epsilon = epsilon
+        @min_points = min_points
+        @names_similarity = names_similarity
+        @gtfs_stops = create_stops_merged
+        clusterize_stops_csv(@gtfs_stops)
+      end
+    end
+
+    def create_stops_merged
+      gtfs_stops = []
+      @gtfs_paths.each do |gtfs_path|
+        gtfs = GTFS::Source.build(gtfs_path)
+        gtfs_stops << gtfs.stops
+      end
+      gtfs_stops.flatten
+    end
+
+    def clusterize_stops_csv(stops_merged)
+      data = import_stops_data(stops_merged, @stops_config_path)
+      @clusters = DBSCAN( data[:stops_data], data[:stops_redis_geodata], :epsilon => @epsilon, :min_points => @min_points, :similarity => @names_similarity, :distance => :haversine_distance2, :labels => data[:stops_names] )
+
+      @clusters.each do |cluster_id, cluster|
+        cluster.each do |stop|
+          gtfs_stop = @gtfs_stops.find { |e| e.lat == stop[:stop_lat] && e.lon == stop[:stop_lon] }
+          stop[:stop_id] = gtfs_stop.id
+          stop[:stop_code] = gtfs_stop.code
+          stop[:parent_station] = gtfs_stop.parent_station
+        end
+      end
+
+      output_path = 'stop_clusters.txt'
+      File.open(output_path, 'w') do |file|
+        @clusters.each do |cluster_id, cluster |
+          file.puts "Cluster #{cluster_id}"
+          cluster.each do |point|
+            file.puts point.inspect
+          end
+          file.puts
+        end
+      end
+    end
+  end
+
+  def gtfs_stops_clusters(gtfs_urls, epsilon, min_points, names_similarity = 1, stop_config_path = '')
+    @gtfs_stops_clustering = GtfsStopsClustering.new(gtfs_urls, epsilon, min_points, names_similarity, stop_config_path)
+    @gtfs_stops_clustering.clusters
+  end
+end
+
+include GtfsStopsClustering

metadata

@@ -0,0 +1,158 @@
+--- !ruby/object:Gem::Specification
+name: gtfs_stops_clustering
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- Visco01
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2023-12-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: distance_measures
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.6
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.6
+- !ruby/object:Gem::Dependency
+  name: text
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.3.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.3.1
+- !ruby/object:Gem::Dependency
+  name: geocoder
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.8.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.8.2
+- !ruby/object:Gem::Dependency
+  name: csv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.2.8
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.2.8
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 5.0.8
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 5.0.8
+- !ruby/object:Gem::Dependency
+  name: gtfs
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.4.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.4.1
+description: A gem to read GTFS stops data and create clusters based on coordinates
+  and stop names' similarities.
+email:
+- visconti373@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/gtfs_stops_clustering.rb
+- lib/gtfs_stops_clustering/data_import.rb
+- lib/gtfs_stops_clustering/dbscan.rb
+- lib/gtfs_stops_clustering/redis_geodata.rb
+- lib/gtfs_stops_clustering/version.rb
+homepage:
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.6.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.4.10
+signing_key:
+specification_version: 4
+summary: A gem to read GTFS stops data and create clusters based on coordinates and
+  stop names' similarities.
+test_files: []