fluent-plugin-mysql-appender 0.2.7 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 052d1cfb0186295c759f1fef124f7d3684d08b96
-  data.tar.gz: ad5182d32736622d1f9b3ebb02c159cdddf26a30
+  metadata.gz: 1afa4ae7160361a720f0ada38c03abf549b017de
+  data.tar.gz: 5d166fe972de1046026b596318ac142b61a22e46
 SHA512:
-  metadata.gz: dc195881e59b37a1c973af70ea1919676702fb1e159377c8f731609155fcca3031f2c013cd05311c27f7e8ab02d8d45b4ebfd6827d3c2e3efe3c84543bbeaa3e
-  data.tar.gz: 18d94dc35249cec9d0014e21e707320668d60f01e56ddf99300cbc963158c11b6eadd983545410928d9c30418ce76ec0729bbef4d6eb8e4a3f0115e4678e9511
+  metadata.gz: dd56bd479c62a0bef34cc5443c3383b6675e6549a0adf73906e79a7a33216e9a5329c99206c468bb81c945102368a20034d418637b375c20512a9f89fd6221b9
+  data.tar.gz: 1074dd4909f86589bccf2ca740feaba86e3d5d66c0d90670d9842dfc439712072632a5565647a30bb01b0ea11e1c61e58e9cb60732ef16a3cc14d10e1992b76f

data/Gemfile

@@ -1,4 +1,4 @@
 source 'https://rubygems.org'
 
-# Specify your gem's dependencies in fluent-plugin-mysql-appender.gemspec
+# Specify your gem's dependencies in fluent-plugin-mysql-replicator.gemspec
 gemspec

data/{LICENSE.txt → LICENSE}

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2016 tsuyoshi_terasaki
+Copyright (c) 2016 TERASAKI Tsuyoshi
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -1,41 +1,83 @@
-# Fluent::Plugin::Mysql::Appender
+# fluent-plugin-mysql-appender
 
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/fluent/plugin/mysql/appender`. To experiment with that code, run `bin/console` for an interactive prompt.
+## Overview
 
-TODO: Delete this and the text above, and describe your gem
+Fluentd input plugin to track insert event only from MySQL database server.
+Simple incremental id's insert.
 
 ## Installation
 
-Add this line to your application's Gemfile:
+install with gem or fluent-gem command as:
 
-```ruby
-gem 'fluent-plugin-mysql-appender'
-```
+`````
+# for system installed fluentd
+$ gem install fluent-plugin-mysql-appender
 
-And then execute:
+# for td-agent2
+$ td-agent-gem install fluent-plugin-mysql-appender
+`````
 
-    $ bundle
+## Included plugins
 
-Or install it yourself as:
+* Input Plugin: mysql_appender
+* Input Plugin: mysql_appender_multi
 
-    $ gem install fluent-plugin-mysql-appender
+## Output example
 
-## Usage
+It is a example when detecting insert events.
 
-TODO: Write usage instructions here
+### sample query
 
-## Development
+`````
+$ mysql -e "create database myweb"
+$ mysql myweb -e "create table search_test(id int auto_increment, text text, PRIMARY KEY (id))"
+$ sleep 10
+$ mysql myweb -e "insert into search_test(text) values('aaa')"
+`````
 
-After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+### result
 
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+`````
+$ tail -f /var/log/td-agent/td-agent.log
+2013-11-25 18:22:25 +0900 appender.myweb.search_test: {"id":"1","text":"aaa"}
+`````
 
-## Contributing
+mysql query log is below
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/fluent-plugin-mysql-appender.
+`````
+$ tail -f /var/log/mysql/general-query.log
+161108 19:25:52        4 Connect    root@localhost on myweb
+            4 Query    SELECT id, text FROM search_test where id > -1 order by id asc
+            4 Quit
+161108 19:26:02        5 Connect    root@localhost on myweb
+            4 Query    SELECT id, text FROM search_test where id > 1 order by id asc
+            4 Quit
+`````
 
+## Tutorial
 
-## License
+### mysql_appender
 
-The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+see example/mysql_single_table_to_tresure_data.md.
+
+**Features**
+
+* Table (or view table) synchronization supported.
+* Replicate small record under a millons table.
+
+### mysql_appender_multi
+
+see example/mysql_multi_table_to_tresure_data.md.
+
+**Features**
+
+* table (or view table) synchronization supported.
+* Multiple table synchronization supported and its DSN stored in yaml file.
+
+## TODO
+
+Pull requests are very welcome like below!!
+
+* more documents
+* more tests.
 

data/example/mysql_multi_table_to_treasure_data.md

@@ -0,0 +1,65 @@
+## case study
+
+It is a guide to replicate multiple mysql table to treasure data.
+
+## configuration
+
+```
+<source>
+  type mysql_appender_multi
+
+  # Set connection settings for replicate source.
+  host localhost
+  username your_mysql_user
+  password your_mysql_password
+  database myweb
+
+  interval 1m  # execute query interval (default: 1m)
+  yaml_path "in_tables.yml"
+</source>
+
+<match appender_multi.*.*>
+  type tdlog
+  endpoint your_td_endpoint
+  apikey your_td_apikey
+
+  auto_create_table
+  buffer_type file
+  buffer_path /var/log/td-agent/buffer/td
+  flush_interval 1m
+  use_ssl true
+  num_threads 8
+
+  <secondary>
+    @type file
+    path /var/log/td-agent/failed_records
+    compress gzip
+  </secondary>
+</match>
+```
+
+Sample "in_tables.yml" is below.
+
+```
+- table_name: test_tbl1
+  primary_key: id
+  time_column: created_at
+  limit: 1000
+  columns:
+    - id
+    - column1
+    - column2
+  last_id: -1
+  td_database: sample_datasets
+
+- table_name: test_tbl2
+  primary_key: id
+  time_column: created_at
+  limit: 1000
+  columns:
+    - id
+    - column1
+    - column2
+  last_id: -1
+  td_database: sample_datasets
+```

data/example/mysql_single_table_to_treasure_data.md

@@ -0,0 +1,49 @@
+## case study
+
+It is a guide to replicate single mysql table to treasure data.
+
+## configuration
+
+```
+<source>
+  type mysql_appender
+
+  # Set connection settings for replicate source.
+  host localhost
+  username your_mysql_user
+  password your_mysql_password
+  database myweb
+
+  # Set replicate query configuration.
+  query SELECT id, text, updated_at from search_test;
+  primary_key id # specify incremental unique key (default: id)
+  interval 1m  # execute query interval (default: 1m)
+
+  # Format output tag for each events.
+  tag appender.myweb.your_td_database.your_td_table
+
+  time_column created_at # specify TIME column.
+  limit   1000 # query limit
+  last_id -1   # specify primary_key start
+</source>
+
+<match appender.*.*>
+  type tdlog
+  endpoint your_td_endpoint
+  apikey your_td_apikey
+
+  auto_create_table
+  buffer_type file
+  buffer_path /var/log/td-agent/buffer/td
+  flush_interval 1m
+  use_ssl true
+  num_threads 8
+
+  <secondary>
+    @type file
+    path /var/log/td-agent/failed_records
+    compress gzip
+  </secondary>
+</match>
+```
+

data/fluent-plugin-mysql-appender.gemspec

@@ -1,13 +1,13 @@
 # coding: utf-8
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-mysql-appender"
-  spec.version       = "0.2.7"
+  spec.version       = "0.3.0"
   spec.authors       = ["TERASAKI Tsuyoshi"]
   spec.email         = ["tsuyoshi_terasaki@realworld.jp"]
 
   spec.summary       = %q{Fluentd input plugin to insert from MySQL database server.}
   spec.description   = %q{Simple incremental id's insert.}
-  spec.homepage      = "https://github.com/rw-hub/fluent-plugin-mysql-appender"
+  spec.homepage      = "https://github.com/tetuyosi/fluent-plugin-mysql-appender"
   spec.license       = "MIT"
 
   spec.files         = `git ls-files`.split($\)

data/lib/fluent/plugin/in_mysql_appender_multi.rb

@@ -32,7 +32,7 @@ module Fluent
         raise Fluent::ConfigError, "mysql_appender_multi: missing 'yaml_path' parameter or file not found."
       end
 
-      if  !File.exist?(@yaml_path)
+      if !File.exist?(@yaml_path)
         raise Fluent::ConfigError, "mysql_appender_multi: 'yaml_path' No such file."
       end
 

data/test/helper.rb

@@ -27,4 +27,4 @@ require 'fluent/plugin/in_mysql_appender'
 require 'fluent/plugin/in_mysql_appender_multi'
 
 class Test::Unit::TestCase
-end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mysql-appender
 version: !ruby/object:Gem::Version
-  version: 0.2.7
+  version: 0.3.0
 platform: ruby
 authors:
 - TERASAKI Tsuyoshi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-11-15 00:00:00.000000000 Z
+date: 2016-11-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -76,15 +76,17 @@ files:
 - ".gitignore"
 - ".travis.yml"
 - Gemfile
-- LICENSE.txt
+- LICENSE
 - README.md
 - Rakefile
+- example/mysql_multi_table_to_treasure_data.md
+- example/mysql_single_table_to_treasure_data.md
 - fluent-plugin-mysql-appender.gemspec
 - lib/fluent/plugin/in_mysql_appender.rb
 - lib/fluent/plugin/in_mysql_appender_multi.rb
 - test/helper.rb
 - test/plugin/test_in_mysql_appender.rb
-homepage: https://github.com/rw-hub/fluent-plugin-mysql-appender
+homepage: https://github.com/tetuyosi/fluent-plugin-mysql-appender
 licenses:
 - MIT
 metadata: {}