elasticsearch_mysql_importer 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5272f35fb0aa318d58f9cbacb9c70ea5ac2ed2d5
-  data.tar.gz: 512a335ab41175198e66836c80b175f82141778e
+  metadata.gz: 0fc769852e9c3fe18f682d2ff00bf490355b6e7a
+  data.tar.gz: 53258865a42da4e001c437e819bacd9faecc16f7
 SHA512:
-  metadata.gz: 7a7e84cbe1b85da430b3e61792535b7ac49c1c920b90bf6a3b7036dba76bfa97e3c3760b08748f4218ac282a78b2dd3e7cff845eb1f98e0388d4185800c454ee
-  data.tar.gz: 3d925a7f1d94bb3a69bd8f72fb8592cd2e98adc5eb4c6c26a119f31722cbc6b48004c32434ab7107909b0f5c3a1f9486fd25715ce608775bd159e9fec240aeb2
+  metadata.gz: aaa6bcc65f4d711bc98afd9b7e58894fcde315a527d1ba9483900e798faa25c30a977f82ea784ac9766aac3463b1e48e40dc285e993cfac992d3a5090c49fc9b
+  data.tar.gz: cad0f17c9e96f98bf6d145e17361ace9b0e76f76a901407d24f1695e6bf20365fde65cc3b338347ea23d94d2eb12bea7a54a90cb80d00ea574fef049f75ae6c1

data/README.md

@@ -1,6 +1,6 @@
 # elasticsearch_mysql_importer
 
-It is importing from mysql table with SQL to elasticsearch not only that, it could generating nested documents.
+It is importing from mysql table with SQL to elasticsearch. Not only that, it could generating nested documents.
 
 ## Usage
 
@@ -13,17 +13,18 @@ It is importing from mysql table with SQL to elasticsearch not only that, it cou
     $ vim example.rb
     
     # Execute script, then it outputs result into ./requests.json
-    $ bundle exec ruby example.rb 
+    $ bundle exec ruby example/example.rb 
     
-    # Index document for elasticsearch
-    $ curl -s -XPOST localhost:9200/_bulk --data-binary @requests.json
+    # Index document for elasticsearch if you didn't call 'write_elasticsearch' in example.rb
+    $ curl -s -XPOST localhost:9200/_bulk --data-binary @example/requests.json
 
 ## TODO
 
 Pull requests are very welcome!!
 
+* add test
 * support thread
-* call elasticsearch bluk api directory
+* support CLI command
 
 ## Contributing
 

data/example/example.rb

@@ -0,0 +1,62 @@
+# coding: utf-8
+require 'elasticsearch_mysql_importer'
+
+importer = ElasticsearchMysqlImporter::Importer.new
+importer.configure do |config|
+  # required
+  config.mysql_host = 'localhost'
+  config.mysql_username = 'your_mysql_username'
+  config.mysql_password = 'your_mysql_password'
+  config.mysql_database = 'some_database'
+
+  # optional, but it is required only generating nested documents
+  config.prepared_query = '
+    CREATE TEMPORARY TABLE tmp_member_skill
+    SELECT 
+      members.id AS member_id,
+      skills.name AS skill_name,
+      skills.url AS skill_url
+    FROM
+      members
+      LEFT JOIN member_skill_relation ON members.id = member_id
+      LEFT JOIN skills ON skills.id = skill_id;
+  '
+  # required for importing into elasticsearch
+  config.query = '
+    SELECT
+      members.id AS member_id,
+      members.name AS member_name,
+      "SELECT skill_name, skill_url FROM tmp_member_skill WHERE member_id = ${member_id}" AS skills
+    FROM
+      members
+    ;
+  '
+  # required for using unique index into elasticsearch
+  config.primary_key = 'member_id'
+
+  # To post index directory into elasticsearch,
+  # configure following two lines and call 'write_elasticsearch' method.
+  # config.elasticsearch_host = 'localhost' # default: localhost
+  # config.elasticsearch_port = 9200 # default: 9200
+
+  # required for specifying elasticsearch index and type
+  config.elasticsearch_index = 'importer_example'
+  config.elasticsearch_type = 'member_skill'
+
+  # required for writing output file path
+  config.output_file = 'example/requests.json'
+end
+
+if importer.write_file
+  puts "Finished to run importer.write_file."
+  puts "The output file is written at '#{importer.output_file}'"
+  puts "Let's try importing file with following curl command."
+  puts "e.g.) curl -s -XPOST localhost:9200/_bulk --data-binary @#{importer.output_file}\n\n"
+end
+
+#if importer.write_elasticsearch 
+#  puts "Finished to run importer.write_elasticsearch."
+#  puts "Let's checking results of index with following curl command."
+#  puts "e.g.) curl localhost:9200/importer_example/_search?pretty=1"
+#end
+# To post index directory into elasticsearch, uncommented out following line.

data/example/example_table.sql

@@ -0,0 +1,33 @@
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `member_skill_relation` (
+  `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
+  `member_id` int(11) DEFAULT NULL,
+  `skill_id` int(11) DEFAULT NULL,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+/*!40101 SET character_set_client = @saved_cs_client */;
+INSERT INTO `member_skill_relation` VALUES (1,1,1),(2,1,2),(3,2,3),(4,3,3),(5,3,4),(6,3,5);
+
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `members` (
+  `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
+  `name` varchar(100) DEFAULT NULL,
+  `created_at` datetime DEFAULT NULL,
+  `updated_at` datetime DEFAULT NULL,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+/*!40101 SET character_set_client = @saved_cs_client */;
+INSERT INTO `members` VALUES (1,'User-A','2014-04-01 15:20:22','2014-04-01 15:20:22'),(2,'User-B','2014-04-01 15:21:30','2014-04-01 15:21:30'),(3,'User-C','2014-04-01 15:21:41','2014-04-01 15:21:41');
+
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `skills` (
+  `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
+  `name` varchar(11) DEFAULT NULL,
+  `url` varchar(250) DEFAULT NULL,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+/*!40101 SET character_set_client = @saved_cs_client */;
+INSERT INTO `skills` VALUES (1,'PHP','http://php.net/'),(2,'Ruby','https://www.ruby-lang.org/'),(3,'Python','https://www.python.org/'),(4,'Java','https://www.java.com/'),(5,'Perl','http://www.perl.org/');

data/example/requests.json

@@ -0,0 +1,6 @@
+{"index":{"_index":"importer_example","_type":"member_skill","_id":"1"}}
+{"member_id":"1","member_name":"User-A","skills":[{"skill_name":"PHP","skill_url":"http://php.net/"},{"skill_name":"Ruby","skill_url":"https://www.ruby-lang.org/"}]}
+{"index":{"_index":"importer_example","_type":"member_skill","_id":"2"}}
+{"member_id":"2","member_name":"User-B","skills":[{"skill_name":"Python","skill_url":"https://www.python.org/"}]}
+{"index":{"_index":"importer_example","_type":"member_skill","_id":"3"}}
+{"member_id":"3","member_name":"User-C","skills":[{"skill_name":"Python","skill_url":"https://www.python.org/"},{"skill_name":"Java","skill_url":"https://www.java.com/"},{"skill_name":"Perl","skill_url":"http://www.perl.org/"}]}

data/lib/elasticsearch_mysql_importer/configuration.rb

@@ -2,6 +2,7 @@ module ElasticsearchMysqlImporter
   class Configuration
     attr_accessor :mysql_host, :mysql_port, :mysql_socket, :mysql_username, :mysql_password, :mysql_encoding
     attr_accessor :mysql_database, :mysql_options, :prepared_query, :query, :primary_key, :output_file
+    attr_accessor :elasticsearch_host, :elasticsearch_port, :elasticsearch_index, :elasticsearch_type
 
     def initialize
       super
@@ -14,6 +15,8 @@ module ElasticsearchMysqlImporter
       @mysql_encoding = 'utf8'
       @mysql_options = { :cast => false, :cache_rows => true }
       @primary_key = 'id'
+      @elasticsearch_host = 'localhost'
+      @elasticsearch_port = 9200
     end
   end
 end

data/lib/elasticsearch_mysql_importer/importer.rb

@@ -22,10 +22,17 @@ module ElasticsearchMysqlImporter
       create_import_file
     end
 
+    def write_elasticsearch
+      call_elasticsearch_bulk_api
+    end
+
     private
     def validate_configuration
       if @configuration.mysql_database.nil? or @configuration.query.nil?
-        raise "Missing Configuration: 'mysql_database' or 'query' are required."
+        raise "Missing Configuration: 'mysql_database' and 'query' are required."
+      end
+      if @configuration.elasticsearch_index.nil? or @configuration.elasticsearch_type.nil?
+        raise "Missing Configuration: 'elasticsearch_index' and 'elasticsearch_type' are required."
       end
     end
 
@@ -38,7 +45,7 @@ module ElasticsearchMysqlImporter
           :username => @configuration.mysql_username,
           :password => @configuration.mysql_password,
           :database => @configuration.mysql_database,
-          :encoding => 'utf8',
+          :encoding => @configuration.mysql_encoding,
           :reconnect => true
         })
       else
@@ -48,7 +55,7 @@ module ElasticsearchMysqlImporter
           :username => @configuration.mysql_username,
           :password => @configuration.mysql_password,
           :database => @configuration.mysql_database,
-          :encoding => 'utf8',
+          :encoding => @configuration.mysql_encoding,
           :reconnect => true
         })
       end
@@ -65,27 +72,45 @@ module ElasticsearchMysqlImporter
     end
 
     def create_import_file
-      file = get_file_io_object
-      db = connect_db
-      db.query(@configuration.prepared_query, @configuration.mysql_options)
-      db.query(@configuration.query, @configuration.mysql_options).each do |row|
-        row.select {|k, v| v.to_s.strip.match(/^SELECT/i) }.each do |k, v|
-          row[k] = [] unless row[k].is_a?(Array)
-          db.query(v.gsub(/\$\{([^\}]+)\}/) {|matched| row[$1].to_s}).each do |nest_row|
-            row[k] << nest_row
+      begin
+        file = get_file_io_object
+        db = connect_db
+        db.query(@configuration.prepared_query, @configuration.mysql_options)
+        db.query(@configuration.query, @configuration.mysql_options).each do |row|
+          row.select {|k, v| v.to_s.strip.match(/^SELECT/i) }.each do |k, v|
+            row[k] = [] unless row[k].is_a?(Array)
+            db.query(v.gsub(/\$\{([^\}]+)\}/) {|matched| row[$1].to_s}).each do |nest_row|
+              row[k] << nest_row
+            end
           end
-        end
-        header = { 
-          "index" => {
-            "_index" => @configuration.elasticsearch_index,
-            "_type" => @configuration.elasticsearch_type,
-            "_id" => row[@configuration.primary_key]
+          header = { 
+            "index" => {
+              "_index" => @configuration.elasticsearch_index,
+              "_type" => @configuration.elasticsearch_type,
+              "_id" => row[@configuration.primary_key]
+            }
           }
-        }
-        file.puts(Yajl::Encoder.encode(header))
-        file.puts(Yajl::Encoder.encode(row))
+          file.puts(Yajl::Encoder.encode(header))
+          file.puts(Yajl::Encoder.encode(row))
+        end
+        file.seek 0
+        return file.path
+      rescue StandardError => e
+        puts "Failed to generate import file: #{e.message}"
+      end
+    end
+
+    def call_elasticsearch_bulk_api
+      begin
+        elasticsearch_bulk_uri = "http://#{@configuration.elasticsearch_host}:#{@configuration.elasticsearch_port}/_bulk"
+        uri = URI.parse(elasticsearch_bulk_uri)
+        data = File.open(@output_file, 'r').read
+        raise "Error: generated import file is empty." if data.empty?
+        http = Net::HTTP.new(uri.host, uri.port)
+        response, body = http.post(uri.path, data, {'Content-type'=>'application/json'})
+      rescue Timeout::Error, StandardError => e
+        puts "Failed to call Bulk API: #{e.message}"
       end
-      return file.path
     end
   end
 end

data/lib/elasticsearch_mysql_importer/version.rb

@@ -1,3 +1,3 @@
 module ElasticsearchMysqlImporter
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: elasticsearch_mysql_importer
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Kentaro Yoshida
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-05-23 00:00:00.000000000 Z
+date: 2014-05-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -79,7 +79,9 @@ files:
 - README.md
 - Rakefile
 - elasticsearch_mysql_importer.gemspec
-- example.rb
+- example/example.rb
+- example/example_table.sql
+- example/requests.json
 - lib/elasticsearch_mysql_importer.rb
 - lib/elasticsearch_mysql_importer/configuration.rb
 - lib/elasticsearch_mysql_importer/importer.rb

data/example.rb

@@ -1,43 +0,0 @@
-# coding: utf-8
-require 'elasticsearch_mysql_importer'
-
-importer = ElasticsearchMysqlImporter::Importer.new
-importer.configure do |config|
-  # required
-  config.mysql_host = 'localhost'
-  config.mysql_username = 'your_mysql_username'
-  config.mysql_password = 'your_mysql_password'
-  config.mysql_database = 'some_database'
-
-  # optional, but it is required only generating nested documents
-  config.prepared_query = '
-    CREATE TEMPORARY TABLE tmp_member_skill
-    SELECT 
-      members.id AS member_id,
-      skills.name AS skill_name,
-      skills.url AS skill_url
-    FROM
-      members
-      LEFT JOIN member_skill_relation ON members.id = member_id
-      LEFT JOIN skills ON skills.id = skill_id;
-  '
-  # required
-  config.query = '
-    SELECT
-      members.id AS member_id,
-      members.name AS member_name,
-      "SELECT skill_name, skill_url FROM tmp_member_skill WHERE member_id = ${member_id}" AS skills,
-      current_timestamp
-    FROM
-      members
-    ;
-  '
-  # required for using unique index for elasticsearch
-  config.primary_key = 'member_id'
-
-  # required for outputs file path
-  config.output_file = 'requests.json'
-end
-
-importer.write_file
-p importer.output_file