rgraphum 0.0.1.alpha

data/lib/rgraphum/analyzer/meme_tracker.rb

@@ -0,0 +1,296 @@
+# -*- coding: utf-8 -*-
+
+class Rgraphum::Analyzer::MemeTracker
+  attr_accessor :distance_max_limit
+  attr_accessor :graph
+  attr_accessor :clusters
+
+  def initialize(graph=Rgraphum::Graph.new)
+    @distance_max_limit = 5
+    self.graph = graph
+  end
+
+  def edit_distance(words_a, words_b, limit=@distance_max_limit)
+    a = words_a.dup
+    b = words_b.dup
+
+    return nil if (a - b | b - a).size  > (limit * 2)
+    d = find_shift_distance(a, b)
+  end
+
+  def find_shift_distance(words_a, words_b, depth=0)
+    return nil if depth > @distance_max_limit
+
+    return words_b.size if words_a.empty?
+    return words_a.size if words_b.empty?
+
+    shifted_words_a = words_a[1..-1]
+    shifted_words_b = words_b[1..-1]
+
+    if words_a[0] == words_b[0]
+      return find_shift_distance(shifted_words_a, shifted_words_b, depth)
+    else
+      depth += 1
+      distance = 1
+      distance_a = find_shift_distance(words_a, shifted_words_b, depth)
+      distance_b = find_shift_distance(shifted_words_a, words_b, depth)
+      distance_c = find_shift_distance(shifted_words_a, shifted_words_b, depth)
+      if delta_distance = [distance_a, distance_b, distance_c].compact.min
+        return distance += delta_distance
+      else
+        return nil
+      end
+    end
+  end
+
+  ###############################################################
+
+  def phrase_clusters
+    new_graph = @graph.dup
+    graph_start_root_vertices = start_root_vertices(new_graph)
+    graph_end_root_vertices   = end_root_vertices(new_graph)
+
+    clusters, cluster_keys = [], []
+    graph_start_root_vertices.each do |graph_start_root_vertex|
+      cluster = build_cluster(graph_start_root_vertex)
+      clusters << cluster
+      cluster_keys << cluster.paths.map { |path| path.end_vertex }
+    end
+    cluster_keys = vertex_id_map(cluster_keys)
+
+    sets = {}
+    clusters.each_with_index do |end_path, i|
+      cluster_keys.each do |end_path_keys|
+        unless (end_path.paths.map { |path| path.end_vertex } & end_path_keys).empty?
+          sets[end_path_keys] ||= []
+          sets[end_path_keys] << graph_start_root_vertices[i]
+          break
+        end
+      end
+    end
+
+    sets = sets.map{ |end_path_keys, end_path_start_root_vertices|
+      [end_path_start_root_vertices, end_path_keys]
+    }
+
+    clusters = []
+    sets.each do |end_path_start_root_vertices, end_path_keys|
+      end_path_start_root_vertices_pt = end_path_start_root_vertices.permutation
+      end_path_keys_pt = end_path_keys.repeated_permutation(end_path_start_root_vertices.size)
+      communities_set = []
+      end_path_start_root_vertices_pt.each_with_index do |end_path_start_root_vertices_p, i|
+        end_path_keys_pt.each_with_index do |end_path_keys_p, j|
+          communities_set << make_communities(end_path_start_root_vertices_p, end_path_keys_p)
+        end
+      end
+
+      sigma_in_sizes = communities_set.map { |communities| sum_sigma_in(communities) }
+      max = sigma_in_sizes.max
+      index = sigma_in_sizes.index(max)
+
+      clusters += communities_set[index]
+    end
+    clusters
+  end
+
+  def vertex_id_map(cluster_keys)
+    return cluster_keys if cluster_keys.size < 2
+    id_map = cluster_keys.dup
+
+    cluster_keys.combination(2).each do |a, b|
+      unless (a & b).empty?
+        id_map.delete(a)
+        id_map.delete(b)
+        id_map << (a | b)
+      end
+    end
+
+    if id_map.size == cluster_keys.size
+      cluster_keys
+    else
+      vertex_id_map(id_map)
+    end
+  end
+
+  def sum_sigma_in(communities)
+    communities.inject(0) { |size, community|
+      size + community.sigma_in
+    }
+  end
+
+  def make_communities(graph_start_root_vertices, graph_end_root_vertices)
+    hashed_cluster = {}
+    used_vertices = {}
+
+    pair = [graph_start_root_vertices, graph_end_root_vertices].transpose
+
+    pair.each do |start_vertex, end_vertex|
+      cluster, used_vertices = find_cluster_with_used_vertices(start_vertex, end_vertex, used_vertices)
+
+      if cluster
+        if hashed_cluster[end_vertex.id]
+          hashed_cluster[end_vertex.id] = (hashed_cluster[end_vertex.id] | cluster)
+        else
+          hashed_cluster[end_vertex.id] = cluster
+        end
+      end
+    end
+
+    communities = hashed_cluster.map do |end_vertex_id, vertices|
+      Rgraphum::Community.new(vertices: vertices)
+    end
+
+    Rgraphum::Communities(communities)
+  end
+
+  # NOTE cluster を探しているっぽい
+  def find_cluster(start_vertex, end_vertex)
+    cluster, used_vertices = find_cluster_with_used_vertices(start_vertex, end_vertex, {})
+    cluster
+  end
+
+  def find_cluster_with_used_vertices(start_vertex, end_vertex, used_vertices)
+    # FIXME rename cluster
+    if used_vertex = used_vertices[start_vertex]
+      if used_vertex == end_vertex
+        return [[], used_vertices]
+      else
+        return [nil, used_vertices]
+      end
+    end
+
+    if start_vertex == end_vertex
+      used_vertices[start_vertex] = end_vertex
+      return [[start_vertex], used_vertices]
+    else
+      if start_vertex.out.empty?
+        return nil, used_vertices
+      end
+    end
+
+    cluster = nil
+    start_vertex.out.each do |vertex|
+      deep_cluster, used_vertices = find_cluster_with_used_vertices(vertex, end_vertex, used_vertices)
+
+      if deep_cluster
+        cluster ||= []
+        cluster += deep_cluster
+      end
+    end
+
+    if cluster
+      cluster << start_vertex
+      used_vertices[start_vertex] = end_vertex
+    end
+
+    [cluster, used_vertices]
+  end
+
+  # NOTE 孤立した cluster を探してるかも?
+  def build_cluster(start_vertex, cluster=nil)
+    cluster ||= Rgraphum::Cluster.new
+    start_vertex.out.each do |vertex|
+      next if cluster.have_vertex_in_path?(vertex, start_vertex)
+      if vertex.out.empty?
+        if cluster.have_end_vertex?(vertex)
+          path = cluster.find_path(vertex.id)
+          cluster.append_vertex path, start_vertex
+        else
+          cluster.add_path Rgraphum::Path.new(vertex, [vertex, start_vertex])
+        end
+      else
+        found = cluster.have_vertex?(vertex) && cluster.have_vertex?(start_vertex)
+        next if found
+
+        cluster = build_cluster(vertex, cluster)
+        cluster.each_path do |path|
+          if path.include?(vertex) and !path.include?(start_vertex)
+            cluster.append_vertex path, start_vertex
+          end
+        end
+      end
+    end
+    cluster
+  end
+
+  def start_root_vertices(target_graph=@graph)
+    target_graph.vertices.find_all{ |vertex| vertex.in.empty? and !vertex.out.empty? }
+  end
+
+  def end_root_vertices(target_graph=@graph)
+    target_graph.vertices.find_all{ |vertex| !vertex.in.empty? and vertex.out.empty? }
+  end
+
+  def find_path(target_vertex, vertices=Rgraphum::Vertices.new)
+    return vertices if vertices.include?(target_vertex)
+    return vertices << target_vertex if target_vertex.out.empty?
+    path_vertices = target_vertex.out.inject(vertices) do |vertices, vertex|
+      size = vertices.size
+      vertices = find_path(vertex, vertices)
+      if vertices.size == size
+        edge_to_delete = target_vertex.edges.where(target: vertex).first
+        target_vertex.edges.delete(edge_to_delete)
+      end
+      vertices
+    end
+    path_vertices << target_vertex
+  end
+
+  def make_path_graph(graph=@graph)
+    p "in make path graph" if Rgraphum.verbose?
+    graph = graph.dup
+
+    p "find srn" if Rgraphum.verbose?
+    graph_start_root_vertices = start_root_vertices(graph)
+
+    p "find path and to_graph" if Rgraphum.verbose?
+    graphes = graph_start_root_vertices.map { |vertex| Rgraphum::Vertices.new(find_path(vertex)).to_graph }
+  end
+
+  def cut_edges_with_srn(graph=@graph)
+    new_graph = Rgraphum::Graph.new
+
+    graphes = make_path_graph(graph)
+
+    new_graph.vertices = graphes.map { |g| g.vertices }.flatten
+    new_graph.edges = graphes.map { |g| g.edges }.flatten
+
+    new_graph.compact_with(:id)
+  end
+
+  def count_same_words_vertices(graph=@graph)
+    graph.vertices.combination(2).each do |vertex_a, vertex_b|
+      vertex_a.count = vertex_a.count.to_i + 1 if vertex_a.words == vertex_b.words
+    end
+  end
+
+  def make_edges(graph=@graph)
+    graph.vertices.sort! { |a, b|  a.start.to_f <=> b.start.to_f }
+
+    graph.vertices.combination(2).each_with_index do |pair, i|
+      if pair[1].start and pair[0].end
+        next unless  pair[0].within_term(pair[1])
+      end
+
+      distance = edit_distance(pair[0].words, pair[1].words)
+      next unless distance
+
+      graph.edges << { source: pair[0], target: pair[1], weight: (1.0 / (distance + 1)) }
+    end
+
+    graph.edges
+  end
+
+  def make_graph(phrase_array)
+    @graph = Rgraphum::Graph.new
+    @graph.vertices = phrase_array
+
+    self.count_same_words_vertices(@graph)
+
+    @graph.compact_with(:words, @graph)
+
+    self.make_edges(@graph)
+
+    @graph
+  end
+end

data/lib/rgraphum/analyzer/twitter/rt_at_mark.rb

@@ -0,0 +1,45 @@
+# -*- coding: utf-8 -*-
+
+require 'time'
+
+class Rgraphum::Analyzer::RTAtMark
+
+  class Rgraphum::Vertex
+    field :twits
+  end
+
+  def make_graph(twits)
+    @graph = Rgraphum::Graph.new
+
+    make_vertices(twits)
+    make_edges(twits)
+
+    @graph
+  end
+
+  def make_vertices(twits,graph=@graph)
+    graph.vertices = twits.map{ |twit| { label:twit[7] } }.uniq!
+    twits.each do |twit|
+      vertex = graph.vertices.where(label: twit[7]).first
+      vertex.twits ||= []
+      vertex.twits << twit.compact
+    end
+  end
+
+  def make_edges(twits,graph=@graph)
+    twits.each_with_index do |twit|
+      next unless atmark_screen_name = pickup_screen_name(twit[8])
+      source_vertex = graph.vertices.where(label: atmark_screen_name).first
+      source_vertex = graph.vertices.build(label: atmark_screen_name) unless source_vertex 
+      target_vertex = graph.vertices.where(label: twit[7]).first
+      graph.edges << {source:source_vertex,target:target_vertex,label:twit[8],start:Time.parse(twit[11])}
+    end
+  end
+
+  def pickup_screen_name(text)
+    return nil unless screen_name = text.match(/(^|[^@0-9_a-zA-Z])@[0-9_a-zA-Z]+($|[^@0-9_a-zA-Z])/)
+    return nil unless screen_name = screen_name[0].gsub(/[^0-9_a-zA-Z]/,"").downcase
+    screen_name
+  end
+
+end

data/lib/rgraphum/analyzer.rb

@@ -0,0 +1,8 @@
+# -*- coding: utf-8 -*-
+
+module Rgraphum::Analyzer
+end
+
+require_relative 'analyzer/linear_regression'
+require_relative 'analyzer/meme_tracker'
+require_relative 'analyzer/twitter/rt_at_mark'

data/lib/rgraphum/cluster.rb

@@ -0,0 +1,67 @@
+# -*- coding: utf-8 -*-
+
+class Rgraphum::Cluster
+  include Enumerable
+
+  def initialize(*paths)
+    if paths.empty?
+      @paths = {}
+    else
+      @paths = Hash[paths.map { |path| [path.end_vertex.id, path] }]
+    end
+  end
+
+  def paths
+    @paths.values
+  end
+
+  def each_path
+    if block_given?
+      @paths.each do |id, path|
+        yield path
+      end
+    else
+      to_enum
+    end
+  end
+
+  def add_path(path)
+    @paths[path.end_vertex.id] = path
+  end
+
+  def find_path(end_vertex_id)
+    @paths[end_vertex_id]
+  end
+
+  def append_vertex(path, vertex)
+    @paths[path.end_vertex.id].vertices << vertex
+  end
+
+  def empty?
+    @paths.empty?
+  end
+
+  def have_vertex_in_path?(end_vertex, vertex)
+    path = find_path(end_vertex.id)
+    return unless path
+    path.vertices.include?(vertex)
+  end
+
+  def have_vertex?(vertex)
+    @paths.any? do |id, path|
+      path.vertices.include?(vertex)
+    end
+  end
+
+  def have_end_vertex?(end_vertex)
+    @paths.any? { |id, path| path.end_vertex.id == (end_vertex.id rescue end_vertex) }
+  end
+
+  def to_hash
+    hash = {}
+    @paths.each do |id, path|
+      hash[path.end_vertex] = path.vertices
+    end
+    hash
+  end
+end

data/lib/rgraphum/communities.rb

@@ -0,0 +1,65 @@
+# -*- coding: utf-8 -*-
+
+def Rgraphum::Communities(array)
+  if array.instance_of?(Rgraphum::Communities)
+    array
+  else
+    Rgraphum::Communities.new(array)
+  end
+end
+
+class Rgraphum::Communities < Rgraphum::RgraphumArray
+
+  # Non-Gremlin methods
+
+  # FIXME
+  # def dup
+  # end
+
+  # add community in communities
+  # @param [Hash] community_hash one community, it is hash.
+  # @return [Community] added community.
+  def build(community_hash={})
+    community = new_community(community_hash)
+    original_push_1 community
+    community
+  end
+
+  alias :original_push_1 :<<
+  def <<(community_hash)
+    build(community_hash)
+    self
+  end
+
+  alias :original_push_m :push
+  def push(*community_hashs)
+    community_hashs.each do |community_hash|
+      self << community_hash
+    end
+    self
+  end
+
+  # Called from delete_if, reject! and reject
+  def delete(community_or_id)
+    if community_or_id.is_a?(Rgraphum::Community)
+      target_community = community_or_id
+    else
+      target_community = where(id: community_or_id).first
+    end
+    super(target_community)
+  end
+
+  protected :original_push_1
+  protected :original_push_m
+
+  private
+
+  def new_community(community_hash={})
+    if community_hash.is_a?(Hash)
+      community_hash = community_hash.dup
+      community_hash[:graph] = @graph
+      community_hash[:id] ||= new_id
+    end
+    Rgraphum::Community(community_hash)
+  end
+end

data/lib/rgraphum/community.rb

@@ -0,0 +1,86 @@
+# -*- coding: utf-8 -*-
+
+def Rgraphum::Community(hash_or_community)
+  if hash_or_community.instance_of?(Rgraphum::Community)
+    hash_or_community
+  else
+    Rgraphum::Community.new(hash_or_community)
+  end
+end
+
+class Rgraphum::Community
+  attr_reader :graph
+  attr_reader :id
+  attr_reader :vertices
+
+  def initialize(options={})
+    @id       = options[:id]
+    @graph    = options[:graph]
+    @vertices = []
+
+    if options[:vertices]
+      options[:vertices].each do |vertex|
+        add_vertex vertex
+      end
+    end
+  end
+
+  def add_vertex(vertex)
+    @vertices << vertex
+  end
+
+  def inter_edges
+    return @inter_edges if @inter_edges
+    @inter_edges = []
+    @vertices.combination(2) do |vertex_a, vertex_b|
+      @inter_edges += (vertex_a.edges & vertex_b.edges)
+    end
+    @inter_edges
+  end
+
+  def outer_edges
+    @outer_edges ||= edges - inter_edges
+  end
+
+  def edges
+    @edges ||= Rgraphum::Edges.new(@vertices.map(&:edges).flatten.uniq)
+  end
+
+  def edges_from(community)
+    edges & community.edges || []
+  end
+
+  def degree_weight
+    @vertices.inject(0) { |sum, vertex| sum + vertex.degree_weight }
+  end
+
+  def sigma_in
+    @sigma_in ||= inter_edges.inject(0) { |sum, edge|
+      sum + edge.weight
+    }
+  end
+
+  def update
+    @inter_edges = nil
+    @outer_edges = nil
+    @edges = nil
+    @sigma_tot = nil
+    @sigma_in = nil
+  end
+
+  def neighborhood?
+    raise NotImplementedError
+  end 
+  
+  def merge(other_community)
+    other_community.vertices.each do |vertex|
+      vertex.community_id = self.id
+      @vertices << vertex
+    end
+    self.update
+  end
+
+  def to_graph
+    Rgraphum::Graph.new(vertices: vertices, edges: inter_edges)
+  end
+end

data/lib/rgraphum/cosine_similarity_matrix.rb

@@ -0,0 +1,40 @@
+# -*- coding: utf-8 -*-
+
+# CosineSimilarity
+# calc vector distance with cosine similarity
+# ex. it make equilateral triangle
+# [ [1,1,0],[1,0,1],[1,0,1] ]
+# it's angle is 60. cosine 60 = 0.5,
+# thus outputs is 
+# [[1.0, 0.5, 0.5], [0.5, 1.0, 0.5], [0.5, 0.5, 1.0]]
+#
+class CosineSimilarityMatrix
+  def similarity(matrix)
+    sim_matrix = []
+
+    # calc cosine similarity
+    # @params [Array] matrix array of array
+    # @return [Array] array of array cosine similarity matrix
+    matrix.each_with_index do |row_fix,j|
+      sim_array = []
+      a2_sum = row_fix.inject(0.0){|sum,a| sum + a**2}
+      matrix.each_with_index do |row_move,i|
+        next sim_array << sim_matrix[i][j] if j > i
+        next sim_array << 1.0 if i == j
+
+        b2_sum = 0.0
+        ab_sum = 0.0
+
+        [row_fix,row_move].transpose.each do |a,b|
+          b2_sum += b**2
+          ab_sum += a*b 
+        end
+        sim_array << ab_sum / ( Math.sqrt(a2_sum * b2_sum) )
+      end
+      sim_matrix << sim_array
+    end
+    sim_matrix
+  end
+
+end
+