rabbit-slide-kou-nagoya-rubykaigi-03 2017.2.11.0

data/data/panda_kotatsu.svg

@@ -0,0 +1,78 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Generator: Adobe Illustrator 15.1.0, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<svg version="1.1" id="レイヤー_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px"
+	 y="0px" width="208.583px" height="84.229px" viewBox="0 0 208.583 84.229" enable-background="new 0 0 208.583 84.229"
+	 xml:space="preserve">
+<path fill="#070001" d="M166.844,51.551c0,0,22.529,5.493,27.457,4.416c4.928-1.076,12.331,1.57,12.592,6.651
+	c0.547,10.658-21.068,4.085-40.164-4.369S166.844,51.551,166.844,51.551z"/>
+<path fill="#FBFAE8" stroke="#060001" stroke-miterlimit="10" d="M139.649,21.6c-0.25-1.75-14.215-9.95-27.25-10.25
+	c-21.75-0.5-27.114,20.586-29.25,34c-0.668,4.197-5.213,10.727-7.687,11.82c-3.067,1.357-3.91,3.32-2.447,4.14
+	c2.438,1.366,37.634,7.29,77.384-3.21C138.899,39.35,139.649,21.6,139.649,21.6z"/>
+<path fill="#070001" d="M106.943,12.473c-0.174-2.576-4.785-7.249-7.797-3.365c-3.014,3.884-2.32,5.065-0.377,6.572
+	C100.711,17.186,107.285,17.564,106.943,12.473z"/>
+<path fill="#FBFAE8" stroke="#080103" stroke-width="2" stroke-miterlimit="10" d="M190.399,55.85c6.013-14.03,0-50-25-52
+	s-34,13-36,26s14,34,28,37S187.399,62.85,190.399,55.85z"/>
+<path fill="#070001" d="M86.064,63.631c-7.491-0.821-13.198-1.123-14.084-1.619c-1.463-0.82,0.227-3.556,3.294-4.912
+	c2.244-0.992,4.75-4.125,6.5-8C81.537,52.529,82.887,60.837,86.064,63.631z"/>
+<path fill="#070001" d="M186.399,21.264c4.852-3.508,1.593-14.713-5-14.164c-9.25,0.771-9,6.029-6,10.368
+	C178.898,22.526,183.399,23.433,186.399,21.264z"/>
+<path fill="#070001" d="M196.899,37.6c1.265-2.53-0.22-8.556-5.5-7.5c-2.5,0.5-0.75,3.058-0.75,5.75
+	C190.649,39.1,194.399,42.6,196.899,37.6z"/>
+<path fill="#070001" d="M186.716,52.486c1.392-2.179-1.573-7.141-5.144-6.354s-1.18,5.688,1.06,6.505
+	C184.87,53.454,186.716,52.486,186.716,52.486z"/>
+<path fill="#070001" d="M176.217,36.27c1.641-3.109-1.977-12.587-8.17-10.74c-7.2,2.148-0.514,10.408,1.726,11.225
+	C172.011,37.571,174.765,39.023,176.217,36.27z"/>
+<path fill="#DB698C" d="M159.466,42.644c0.479-0.87,7.156,2.642,13.117,3.996c0.071,4.148,2.923,13.869,2.223,14.086
+	c-1.019,0.474-3.349-4.081-6.556-7.665C165.042,49.479,158.986,43.514,159.466,42.644z"/>
+<path fill="#070001" d="M176.371,47.823c1.012-1.585-1.146-5.194-3.741-4.622c-2.597,0.573-0.857,4.136,0.771,4.731
+	C175.029,48.527,176.371,47.823,176.371,47.823z"/>
+<path fill="#070001" d="M131.899,21.85c-0.5,13,5.417,32.583,15.25,40.75c7.289,6.054,16,7.75,14,13.75s-13.834,7.584-21.833-0.75
+	C132.071,68.053,122.149,34.35,131.899,21.85z"/>
+<g>
+	<path fill="#EDE6CD" d="M115.6,83.201c-3.067,0-12.861-1.058-22.333-2.08c-8.52-0.92-17.329-1.871-19.742-1.871
+		c-1.398,0-4.595,0.29-8.643,0.656c-6.615,0.6-15.674,1.42-22.3,1.42c-3.993,0-6.667-0.293-8.178-0.897
+		C21.229,75.158,2.573,52.935,0.034,43.98c-2.013-7.1,1.647-10.246,5.521-13.576c2.168-1.863,4.625-3.975,6.489-7.048
+		c1.47-2.424,2.82-4.867,4.125-7.23c3.748-6.783,6.708-12.141,10.163-12.77c2.458-0.447,10.359-0.664,24.156-0.664
+		c22.211,0,52.131,0.595,54.616,0.645c0.991-0.292,2.493-0.441,4.469-0.441c6.996,0,20.62,2.044,26.955,5.739
+		c5.885,3.433,11.121,18.022,11.121,27.114c0,7.64,2.998,15.207,9.164,23.136c1.505,1.935,2.045,4.46,1.521,7.111
+		c-0.764,3.86-3.703,7.504-8.063,9.996C141.409,81.057,121.878,83.201,115.6,83.201z"/>
+	<path fill="#50482E" d="M50.489,3.693c23.329,0,54.758,0.648,54.758,0.648c0.901-0.3,2.419-0.444,4.326-0.444
+		c7.353,0,20.496,2.129,26.451,5.603c5.418,3.161,10.625,17.233,10.625,26.25c0,10,5,18.125,9.375,23.75
+		c3.165,4.069,1.45,11.225-6.25,15.625c-8.75,5-28.343,7.076-34.175,7.076S78.628,78.25,73.524,78.25
+		c-3.837,0-20.393,2.076-30.943,2.076c-3.48,0-6.305-0.226-7.807-0.826c-12.5-5-31.25-26.875-33.778-35.792
+		C-1.9,33.491,7.57,32.661,12.899,23.875c5.525-9.11,9.53-18.792,13.612-19.534C29.185,3.855,38.825,3.693,50.489,3.693
+		 M50.492,1.693v2l-0.001-2c-13.859,0-21.819,0.223-24.336,0.68c-3.922,0.713-6.832,5.98-10.859,13.27
+		c-1.3,2.354-2.645,4.788-4.105,7.195c-1.781,2.937-4.071,4.905-6.285,6.808c-3.943,3.389-8.02,6.893-5.833,14.608
+		c2.586,9.119,21.563,31.744,34.96,37.104c1.653,0.661,4.37,0.969,8.549,0.969c6.672,0,15.757-0.823,22.391-1.424
+		c3.863-0.35,7.2-0.652,8.553-0.652c2.359,0,11.142,0.948,19.635,1.865c9.498,1.025,19.319,2.086,22.44,2.086
+		c6.354,0,26.144-2.184,35.167-7.34c4.612-2.636,7.729-6.524,8.548-10.67c0.582-2.939-0.026-5.752-1.711-7.919
+		c-6.025-7.747-8.954-15.113-8.954-22.522c0-9.543-5.36-24.328-11.617-27.978c-6.572-3.833-20.17-5.875-27.459-5.875
+		c-2.002,0-3.512,0.144-4.598,0.438C101.629,2.268,72.335,1.693,50.492,1.693L50.492,1.693z"/>
+</g>
+<path fill="#DFD2B4" d="M0.753,43.987c-2.94-10.371,6.674-11.214,12.083-20.133c5.609-9.248,9.675-19.077,13.818-19.83
+	c0.649-0.118,1.706-0.217,3.096-0.299l21.07,19.707c-8.461,20.641-10.847,34.102-11.506,46.02
+	c-0.309,5.582,2.985,8.004,5.626,10.744c-4.057,0.052-7.831,0.294-9.518-0.381C22.733,74.738,3.319,53.04,0.753,43.987z"/>
+<path fill="#DFD2B4" d="M157.337,62.125c0.738,4.104-1.227,9.636-7.354,13.138c-8.968,5.125-28.66,6.873-34.637,6.929
+	c-3.411,0.032-8.264-0.592-16.27-1.473c23.169-1.239,37.861-4.521,49.712-9.23C153.932,69.442,155.812,65.644,157.337,62.125z"/>
+<path fill="#DB2218" d="M146.645,47.98c-7.629,0.012-10.737-7.677-10.62-13.605c0.087-4.386,4.525-8.732,8.566-10.085
+	c1.273,4.085,2.059,8.34,2.059,11.835c0,4.25,1.001,8.038,2.375,11.563C148.294,47.79,147.447,47.979,146.645,47.98z"/>
+<path fill="#DB2218" d="M124.889,12.367c-2.125-0.755-6.124,2.155-6.183,5.093c-0.058,2.938,1.482,6.749,5.264,6.743
+	c4.604-0.007,6.827-3.49,5.821-7.134S126.056,12.782,124.889,12.367z"/>
+<path fill="#DB2218" d="M118.161,53.44c12.887,0,20.824,16.796,16.107,26.648c-7.217,1.526-15.329,2.113-18.64,2.113
+	c-1.528,0-5.839-0.214-10.419-0.854c-1.315-1.664-2.521-3.419-3.495-4.882C96.781,69.064,103.358,53.44,118.161,53.44z"/>
+<path fill="#DB2218" d="M67.463,74.771c11.481-6.263,8.771-19.138,1.595-24.719c-7.176-5.582-19.089,5.059-19.934,14.353
+	C48.326,73.177,58.692,79.556,67.463,74.771z"/>
+<path fill="#DB2218" d="M102.274,38.875c0-6.374-5.625-12.5-13.125-6.25s0.625,12.5,3.125,13.125S102.274,43.25,102.274,38.875z"/>
+<path fill="#DB2218" d="M54.149,26.375c1.207,5.634-3.125,9.375-7.5,10.625s-19.375-3.125-10.625-16.25
+	C41.929,11.894,52.274,17.625,54.149,26.375z"/>
+<path fill="#C61F1E" d="M7.613,29.643c1.416,0.671,2.786,1.708,4.045,3.196c7.782,9.197,3.864,17.439-4.244,21.223
+	c-0.294,0.137-0.583,0.257-0.87,0.368c-3-4.142-5.167-7.92-5.918-10.569C-1.502,36.355,2.944,33.839,7.613,29.643z"/>
+<path fill="#C61F1E" d="M112.649,82.156c-1.752-0.128-3.811-0.349-3.82-0.349c-1.07-0.101-1.986-0.194-3.523-0.37
+	c-0.197-0.248-0.645-0.791-0.832-1.039c12.78-0.894,22.723-2.367,31.018-4.58c-0.195,1.512-0.633,3.033-1.266,4.355
+	c-0.17,0.107-5.879,1.047-11.999,1.582C119.203,82.02,116.356,82.428,112.649,82.156z"/>
+<path fill="#C61F1E" d="M36.024,20.75c2.061-3.09,4.662-4.4,7.255-4.43l6.37,7.471c-0.894,5.897-2.957,8.532-4.046,13.383
+	C40.32,37.604,27.876,32.973,36.024,20.75z"/>
+<path fill="#815C25" d="M100.165,0.55c4.374,0,29.452,20.08,29.452,24.204c0,2.916-75.819,2.187-80.923,0
+	C44.43,22.926,19.241,3.757,23.617,1.57C28.181-0.712,100.165,0.55,100.165,0.55z"/>
+</svg>

data/examples/data/bow.metadata.tf.filtered

@@ -0,0 +1 @@
+{"n_documents":13181,"n_features":15914}

data/examples/data/bow.metadata.tf.raw

@@ -0,0 +1 @@
+{"n_documents":14066,"n_features":15916}

data/examples/data/bow.metadata.tfidf.filtered

@@ -0,0 +1 @@
+{"n_documents":13181,"n_features":15914}

data/examples/data/bow.metadata.tfidf.raw

@@ -0,0 +1 @@
+{"n_documents":14066,"n_features":15916}

data/examples/estimate-topics.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python
+
+import sys
+sys.path.append("/home/kou/work/cpp/arrow/python")
+
+import json
+
+import scipy as sp
+import pandas as pd
+import pyarrow as A
+from sklearn.decomposition import LatentDirichletAllocation
+
+LDA = LatentDirichletAllocation
+
+metadata_path = sys.argv[1]
+data_path = sys.argv[2]
+topics_path = sys.argv[3]
+if len(sys.argv) >= 5:
+    n_documents = int(sys.argv[4])
+else:
+    n_documents = None
+if len(sys.argv) >= 6:
+    n_topics = int(sys.argv[5])
+else:
+    n_topics = None
+
+if n_topics is None or n_topics == -1:
+    n_topics = 100
+
+with open(metadata_path) as metadata_file:
+    metadata = json.load(metadata_file)
+if n_documents is None or n_documents == -1:
+    n_documents = metadata["n_documents"]
+else:
+    n_documents = min(n_documents, metadata["n_documents"])
+n_features = metadata["n_features"]
+
+lda = LDA(n_topics=n_topics,
+          learning_method="online",
+          total_samples=n_documents,
+          n_jobs=1)
+
+with A.io.MemoryMappedFile(data_path, "rb") as source:
+    reader = A.ipc.StreamReader(source)
+    for i, batch in enumerate(reader):
+        if i >= n_documents:
+            break
+        sys.stdout.write("\r%.3f%%" % ((i / n_documents) * 100))
+        df = batch.to_pandas()
+        corpus = sp.sparse.csr_matrix((df["score"].values,
+                                       df["term_id"].values,
+                                       [0, df["term_id"].size]),
+                                      shape=(1, n_features))
+        lda.partial_fit(corpus)
+sys.stdout.write("\n")
+
+def topic_to_df(topic):
+    n_top_terms = 10
+    return pd.DataFrame([[i, topic[i]]
+                         for i in topic.argsort()[:-n_top_terms - 1:-1]],
+                        columns=["term_id", "score"])
+
+topic = lda.components_[0]
+topic_df = topic_to_df(topic)
+schema = A.RecordBatch.from_pandas(topic_df).schema
+with open(topics_path, "wb") as sink:
+    writer = A.ipc.StreamWriter(sink, schema)
+    for topic in lda.components_:
+        topic_df = topic_to_df(topic)
+        topic_record_batch = A.RecordBatch.from_pandas(topic_df)
+        writer.write_batch(topic_record_batch)
+    writer.close()

data/examples/raw-show-related-terms.rb

@@ -0,0 +1,38 @@
+#!/usr/bin/env ruby
+
+require "groonga"
+require "gi"
+
+db_path = ARGV[0]
+output_path = ARGV[1]
+
+Arrow = GI.load("Arrow")
+ArrowIO = GI.load("ArrowIO")
+ArrowIPC = GI.load("ArrowIPC")
+
+Groonga::Database.open(db_path)
+terms = Groonga["Words"]
+
+input_stream = ArrowIO::MemoryMappedFile.open(output_path, :read)
+begin
+  reader = ArrowIPC::StreamReader.open(input_stream)
+  loop do
+    record_batch = reader.next_record_batch
+    break if record_batch.nil?
+    columns = record_batch.columns
+    related_terms = []
+    previous_score = nil
+    record_batch.n_rows.times do |i|
+      score = columns[1].get_value(i)
+      break if score < 0.1
+      previous_score ||= score
+      break if (previous_score - score) > (score / 2.0)
+      term = Groonga::Record.new(terms, columns[0].get_value(i)).key
+      related_terms << [term, score]
+    end
+    next if related_terms.size < 2
+    p related_terms
+  end
+ensure
+  input_stream.close
+end

data/examples/raw-write-bow.rb

@@ -0,0 +1,124 @@
+#!/usr/bin/env ruby
+
+require "groonga"
+require "gi"
+
+db_path = ARGV[0]
+metadata_output_path = ARGV[1]
+data_output_path = ARGV[2]
+use_tfidf = (ARGV[3] != "tf")
+use_filter = (ARGV[4] != "raw")
+
+Arrow = GI.load("Arrow")
+ArrowIO = GI.load("ArrowIO")
+ArrowIPC = GI.load("ArrowIPC")
+
+Groonga::Database.open(db_path)
+
+Groonga::Schema.define do |schema|
+  schema.create_table("Words",
+                      :type => :patricia_trie,
+                      :key_type => "ShortText",
+                      :default_tokenizer => "TokenMecab",
+                      :normalizer => "NormalizerAuto") do |table|
+    table.index("Entries.document")
+  end
+end
+
+n_entries = Groonga["Entries"].size
+too_many_much_threshold = n_entries * 0.25
+too_less_much_threshold = n_entries * 0.001
+
+bow = {}
+index = Groonga["Words.Entries_document"]
+max_term_id = 0
+index.table.open_cursor(:order_by => :id) do |table_cursor|
+  table_cursor.each do |term|
+    n_match_documents = index.estimate_size(term)
+    # p [term.key, n_match_documents, (n_match_documents / n_entries.to_f)]
+    if use_filter
+      if n_match_documents <= too_less_much_threshold
+        p [:skip, :too_less, term.key, n_match_documents]
+        next
+      end
+      if n_match_documents >= too_many_much_threshold
+        p [:skip, :too_many, term.key, n_match_documents]
+        next
+      end
+    end
+    max_term_id = [max_term_id, term.id].max
+    df = Math.log(n_entries.to_f / n_match_documents)
+    index.open_cursor(term.id,
+                      :with_position => false) do |index_cursor|
+      index_cursor.each(:reuse_posting_object => true) do |posting|
+        next unless posting.record.version.key == "2.4.0"
+        bow[posting.record_id] ||= []
+        if use_tfidf
+          score = posting.term_frequency / df
+        else
+          score = posting.term_frequency
+        end
+        bow[posting.record_id] << [posting.term_id, score]
+      end
+    end
+  end
+end
+
+File.open(metadata_output_path, "w") do |metadata_file|
+  metadata_file.puts({
+                       "n_documents" => bow.size,
+                       "n_features" => max_term_id,
+                     }.to_json)
+end
+
+module Arrow
+  class ArrayBuilder
+    class << self
+      def build(values)
+        builder = new
+        values.each do |value|
+          builder.append(value)
+        end
+        builder.finish
+      end
+    end
+  end
+
+  class UInt32Array
+    class << self
+      def new(values)
+        UInt32ArrayBuilder.build(values)
+      end
+    end
+  end
+
+  class DoubleArray
+    class << self
+      def new(values)
+        DoubleArrayBuilder.build(values)
+      end
+    end
+  end
+end
+
+output_stream = ArrowIO::FileOutputStream.open(data_output_path, false)
+begin
+  term_id_field = Arrow::Field.new("term_id", Arrow::UInt32DataType.new)
+  score_field = Arrow::Field.new("score", Arrow::DoubleDataType.new)
+  schema = Arrow::Schema.new([term_id_field, score_field])
+  writer = ArrowIPC::StreamWriter.open(output_stream, schema)
+  begin
+    bow.each do |record_id, words|
+      term_ids = Arrow::UInt32Array.new(words.collect(&:first))
+      scores = Arrow::DoubleArray.new(words.collect(&:last))
+      record_batch = Arrow::RecordBatch.new(schema,
+                                            words.size,
+                                            [term_ids, scores])
+      writer.write_record_batch(record_batch)
+    end
+  ensure
+    writer.close
+  end
+ensure
+  output_stream.close
+end

data/examples/read.rb

@@ -0,0 +1,30 @@
+#!/usr/bin/env ruby
+
+require "gi"
+
+Arrow = GI.load("Arrow")
+ArrowIO = GI.load("ArrowIO")
+ArrowIPC = GI.load("ArrowIPC")
+
+module Arrow
+  class Array
+    def [](i)
+      get_value(i)
+    end
+
+    include Enumerable
+    def each
+      length.times do |i|
+        yield(self[i])
+      end
+    end
+  end
+end
+
+file = ArrowIO::MemoryMappedFile.open("/tmp/xxx", :read)
+reader = ArrowIPC::FileReader.open(file)
+p reader.schema.fields.collect(&:name)
+record_batch = reader.get_record_batch(0)
+record_batch.n_rows.times do |i|
+  p record_batch.columns.collect {|column| column[i]}
+end

data/examples/run.sh

@@ -0,0 +1,33 @@
+#!/bin/sh
+
+export LD_LIBRARY_PATH=/tmp/local/lib:$LD_LIBRARY_PATH
+export GI_TYPELIB_PATH=/tmp/local/lib/girepository-1.0:$GI_TYPELIB_PATH
+
+base_dir=$(dirname $0)
+data_dir=$base_dir/data
+
+mkdir -p $data_dir
+
+for score in tf tfidf; do
+    for filter in raw filtered; do
+	(
+	    ruby \
+		-I ~/work/ruby/rarrow/lib \
+		-I ~/work/ruby/rroonga/lib \
+		-I ~/work/ruby/rroonga/ext/groonga \
+		$base_dir/write-bow.rb \
+		~/work/ruby/rurema-search/groonga-database/bitclust.db \
+		$data_dir/bow.metadata.$score.$filter \
+		$data_dir/bow.data.$score.$filter \
+		$score \
+		$filter &&
+	    python \
+		$base_dir/estimate-topics.py \
+		$data_dir/bow.metadata.$score.$filter \
+		$data_dir/bow.data.$score.$filter \
+		$data_dir/topics.$score.$filter
+	) &
+    done
+done
+
+wait

data/examples/show-related-terms.rb

@@ -0,0 +1,39 @@
+#!/usr/bin/env ruby
+
+require "groonga"
+require "arrow"
+
+db_path = ARGV[0]
+topics_path = ARGV[1]
+
+Groonga::Database.open(db_path)
+terms = Groonga["Words"]
+index = Groonga["Words.Entries_document"]
+
+Arrow::IO::MemoryMappedFile.open(topics_path, :read) do |input_stream|
+  Arrow::IPC::StreamReader.open(input_stream) do |reader|
+    reader.each do |record_batch|
+      related_terms = []
+      previous_score = nil
+      # p :topic_raw
+      # record_batch.each do |record|
+      #   term = Groonga::Record.new(terms, record["term_id"]).key
+      #   p [record["term_id"], record["score"], term, index.estimate_size(term)]
+      # end
+      record_batch.each do |record|
+        score = record["score"]
+        break if score < 0.1
+        previous_score ||= score
+        break if (previous_score - score) > (previous_score / 2.0)
+        previous_score = score
+        term = Groonga::Record.new(terms, record["term_id"]).key
+        related_terms << [term, score]
+      end
+      next if related_terms.size < 2
+      p :topic
+      related_terms.each do |term|
+        p term
+      end
+    end
+  end
+end