flydata 0.0.5.6 → 0.1.0

data/Gemfile

@@ -6,6 +6,9 @@ gem "activesupport", "~> 4.0.0"
 gem "json", "~> 1.8.0"
 gem "highline", "~> 1.6.19"
 gem "fluentd", "0.10.35"
+gem "ruby-binlog", ">= 1.0.0", :git => "https://github.com/hapyrus/ruby-binlog.git"
+gem "fluent-plugin-mysql-binlog", "~> 0.0.2"
+gem "mysql2", "~> 0.3.11"
 
 group :development do
   gem "bundler"
@@ -14,5 +17,10 @@ group :development do
   #gem "autotest"
   #gem "autotest-standalone"
   #gem "autotest-notification"
+  gem 'timecop'
   gem "sqlite3"
+  gem 'ruby-prof'
+  gem 'activemodel', '~> 4.0.0'
+  gem 'activerecord', '~> 4.0.0'
+  gem 'protected_attributes'
 end

data/Gemfile.lock

@@ -1,6 +1,21 @@
+GIT
+  remote: https://github.com/hapyrus/ruby-binlog.git
+  revision: e381b4fc411ca2c2402aefc25809ac3c97d75f94
+  specs:
+    ruby-binlog (1.0.0)
+
 GEM
   remote: http://rubygems.org/
   specs:
+    activemodel (4.0.0)
+      activesupport (= 4.0.0)
+      builder (~> 3.1.0)
+    activerecord (4.0.0)
+      activemodel (= 4.0.0)
+      activerecord-deprecated_finders (~> 1.0.2)
+      activesupport (= 4.0.0)
+      arel (~> 4.0.0)
+    activerecord-deprecated_finders (1.0.3)
     activesupport (4.0.0)
       i18n (~> 0.6, >= 0.6.4)
       minitest (~> 4.2)
@@ -8,13 +23,18 @@ GEM
       thread_safe (~> 0.1)
       tzinfo (~> 0.3.37)
     addressable (2.3.5)
+    arel (4.0.2)
     atomic (1.1.14)
-    builder (3.2.2)
+    builder (3.1.4)
     cool.io (1.1.1)
       iobuffer (>= 1.0.0)
     diff-lcs (1.2.4)
     faraday (0.8.8)
       multipart-post (~> 1.2.0)
+    fluent-plugin-mysql-binlog (0.0.2)
+      activesupport
+      fluentd
+      kodama
     fluentd (0.10.35)
       cool.io (~> 1.1.0)
       http_parser.rb (~> 0.5.1)
@@ -47,12 +67,15 @@ GEM
     json (1.8.0)
     jwt (0.1.8)
       multi_json (>= 1.5)
+    kodama (0.1.1)
+      ruby-binlog (>= 0.1.9)
     mime-types (1.25)
     minitest (4.7.5)
     msgpack (0.5.6)
     multi_json (1.8.2)
     multi_xml (0.5.5)
     multipart-post (1.2.0)
+    mysql2 (0.3.11)
     nokogiri (1.5.10)
     oauth2 (0.9.2)
       faraday (~> 0.8)
@@ -61,6 +84,8 @@ GEM
       multi_json (~> 1.0)
       multi_xml (~> 0.5)
       rack (~> 1.2)
+    protected_attributes (1.0.3)
+      activemodel (>= 4.0.0, < 5.0)
     rack (1.5.2)
     rake (10.1.0)
     rdoc (4.0.1)
@@ -75,9 +100,11 @@ GEM
     rspec-expectations (2.14.3)
       diff-lcs (>= 1.1.3, < 2.0)
     rspec-mocks (2.14.3)
+    ruby-prof (0.14.2)
     sqlite3 (1.3.8)
     thread_safe (0.1.3)
       atomic
+    timecop (0.7.1)
     tzinfo (0.3.38)
     yajl-ruby (1.1.0)
 
@@ -85,13 +112,21 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
+  activemodel (~> 4.0.0)
+  activerecord (~> 4.0.0)
   activesupport (~> 4.0.0)
   bundler
+  fluent-plugin-mysql-binlog (~> 0.0.2)
   fluentd (= 0.10.35)
   highline (~> 1.6.19)
   i18n (~> 0.6.5)
   jeweler
   json (~> 1.8.0)
+  mysql2 (~> 0.3.11)
+  protected_attributes
   rest-client (~> 1.6.7)
   rspec
+  ruby-binlog (>= 1.0.0)!
+  ruby-prof
   sqlite3
+  timecop

data/VERSION

@@ -1 +1 @@
-0.0.5.6
+0.1.0

data/bin/fdmysqldump

@@ -0,0 +1,59 @@
+#!/usr/bin/env ruby
+
+#require 'optparse'
+
+lib = File.expand_path(File.dirname(__FILE__) + '/../lib')
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+require 'flydata/table_def'
+
+if `which mysqldump`.empty?
+  abort "mysqldump is not installed."
+end
+
+# TODO Implement options instead of passing all to mysqldump.
+=begin
+opts = {}
+
+op = OptionParser.new do |o|
+  o.on('-h', '--host NAME', 'Connect to host.') do |v|
+    opts[:host] = v
+  end
+  o.on('-P', '--port NUMBER', 'Port number to use for connection.') do |v|
+    opts[:port] = v
+  end
+  o.on('-u', '--user NAME', 'User for login if not current user.') do |v|
+    opts[:user] = v
+  end
+  o.on('-p', '--password [TEXT]', "Password to use when connecting to server.  If password is not given it's solicited on the tty.") do |v|
+    opts[:password] = v
+  end
+end
+
+op.parse!
+
+=end
+command = "mysqldump -d"
+
+=begin
+command += " -u#{opts[:user]}" if opts[:user]
+command += " -h#{opts[:host]}" if opts[:host]
+command += " -P#{opts[:port]}" if opts[:port]
+command += " -p#{opts[:password]}" if opts[:password]
+=end
+command += " " + ARGV.join(' ')
+
+IO.popen(command, 'r') do |io|
+  create_flydata_ctl_table = true
+  while !io.eof?
+    mysql_tabledef = Flydata::Sync::MysqlTableDef.create(io)
+    if mysql_tabledef.nil?
+      # stream had no more create table definition
+      break
+    end
+    flydata_tabledef = mysql_tabledef.to_flydata_tabledef
+    puts Flydata::Sync::RedshiftTableDef.from_flydata_tabledef(flydata_tabledef, flydata_ctl_table: create_flydata_ctl_table)
+    create_flydata_ctl_table = false
+  end
+end
+

data/flydata.gemspec

@@ -5,14 +5,14 @@
 
 Gem::Specification.new do |s|
   s.name = "flydata"
-  s.version = "0.0.5.6"
+  s.version = "0.1.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Koichi Fujikawa"]
-  s.date = "2014-02-25"
+  s.date = "2014-04-18"
   s.description = "FlyData Command Line Interface"
   s.email = "sysadmin@flydata.co"
-  s.executables = ["flydata"]
+  s.executables = ["fdmysqldump", "flydata"]
   s.files = [
     ".gitignore",
     ".rspec",
@@ -20,6 +20,7 @@ Gem::Specification.new do |s|
     "Gemfile.lock",
     "Rakefile",
     "VERSION",
+    "bin/fdmysqldump",
     "bin/flydata",
     "flydata.gemspec",
     "lib/fly_data_model.rb",
@@ -27,10 +28,13 @@ Gem::Specification.new do |s|
     "lib/flydata/api/base.rb",
     "lib/flydata/api/data_entry.rb",
     "lib/flydata/api/data_port.rb",
+    "lib/flydata/api/redshift_cluster.rb",
     "lib/flydata/api_client.rb",
     "lib/flydata/cli.rb",
     "lib/flydata/command/base.rb",
+    "lib/flydata/command/conf.rb",
     "lib/flydata/command/crontab.rb",
+    "lib/flydata/command/encrypt.rb",
     "lib/flydata/command/kill_all.rb",
     "lib/flydata/command/login.rb",
     "lib/flydata/command/restart.rb",
@@ -40,26 +44,42 @@ Gem::Specification.new do |s|
     "lib/flydata/command/setup.rb",
     "lib/flydata/command/start.rb",
     "lib/flydata/command/stop.rb",
+    "lib/flydata/command/sync.rb",
+    "lib/flydata/command/version.rb",
     "lib/flydata/credentials.rb",
     "lib/flydata/cron.rb",
+    "lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb",
+    "lib/flydata/fluent-plugins/out_forward_ssl.rb",
+    "lib/flydata/fluent-plugins/preference.rb",
     "lib/flydata/flydata_crontab.sh",
     "lib/flydata/helpers.rb",
     "lib/flydata/heroku.rb",
     "lib/flydata/heroku/configuration_methods.rb",
     "lib/flydata/heroku/instance_methods.rb",
     "lib/flydata/log_monitor.rb",
+    "lib/flydata/preference/data_entry_preference.rb",
     "lib/flydata/proxy.rb",
+    "lib/flydata/sync_file_manager.rb",
+    "lib/flydata/table_def.rb",
+    "lib/flydata/table_def/mysql_table_def.rb",
+    "lib/flydata/table_def/redshift_table_def.rb",
+    "lib/flydata/util/encryptor.rb",
+    "spec/fluent_plugins_spec_helper.rb",
     "spec/fly_data_model_spec.rb",
     "spec/flydata/api/data_entry_spec.rb",
     "spec/flydata/command/sender_spec.rb",
+    "spec/flydata/command/sync_spec.rb",
+    "spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb",
     "spec/flydata/heroku_spec.rb",
+    "spec/flydata/util/encryptor_spec.rb",
     "spec/flydata_spec.rb",
-    "spec/spec_helper.rb"
+    "spec/spec_helper.rb",
+    "tmpl/redshift_mysql_data_entry.conf.tmpl"
   ]
   s.homepage = "http://flydata.co/"
   s.licenses = ["All right reserved."]
   s.require_paths = ["lib"]
-  s.rubygems_version = "1.8.28"
+  s.rubygems_version = "1.8.24"
   s.summary = "FlyData CLI"
 
   if s.respond_to? :specification_version then
@@ -72,10 +92,18 @@ Gem::Specification.new do |s|
       s.add_runtime_dependency(%q<json>, ["~> 1.8.0"])
       s.add_runtime_dependency(%q<highline>, ["~> 1.6.19"])
       s.add_runtime_dependency(%q<fluentd>, ["= 0.10.35"])
+      s.add_runtime_dependency(%q<ruby-binlog>, [">= 1.0.0"])
+      s.add_runtime_dependency(%q<fluent-plugin-mysql-binlog>, ["~> 0.0.2"])
+      s.add_runtime_dependency(%q<mysql2>, ["~> 0.3.11"])
       s.add_development_dependency(%q<bundler>, [">= 0"])
       s.add_development_dependency(%q<jeweler>, [">= 0"])
       s.add_development_dependency(%q<rspec>, [">= 0"])
+      s.add_development_dependency(%q<timecop>, [">= 0"])
       s.add_development_dependency(%q<sqlite3>, [">= 0"])
+      s.add_development_dependency(%q<ruby-prof>, [">= 0"])
+      s.add_development_dependency(%q<activemodel>, ["~> 4.0.0"])
+      s.add_development_dependency(%q<activerecord>, ["~> 4.0.0"])
+      s.add_development_dependency(%q<protected_attributes>, [">= 0"])
     else
       s.add_dependency(%q<rest-client>, ["~> 1.6.7"])
       s.add_dependency(%q<i18n>, ["~> 0.6.5"])
@@ -83,10 +111,18 @@ Gem::Specification.new do |s|
       s.add_dependency(%q<json>, ["~> 1.8.0"])
       s.add_dependency(%q<highline>, ["~> 1.6.19"])
       s.add_dependency(%q<fluentd>, ["= 0.10.35"])
+      s.add_dependency(%q<ruby-binlog>, [">= 1.0.0"])
+      s.add_dependency(%q<fluent-plugin-mysql-binlog>, ["~> 0.0.2"])
+      s.add_dependency(%q<mysql2>, ["~> 0.3.11"])
       s.add_dependency(%q<bundler>, [">= 0"])
       s.add_dependency(%q<jeweler>, [">= 0"])
       s.add_dependency(%q<rspec>, [">= 0"])
+      s.add_dependency(%q<timecop>, [">= 0"])
       s.add_dependency(%q<sqlite3>, [">= 0"])
+      s.add_dependency(%q<ruby-prof>, [">= 0"])
+      s.add_dependency(%q<activemodel>, ["~> 4.0.0"])
+      s.add_dependency(%q<activerecord>, ["~> 4.0.0"])
+      s.add_dependency(%q<protected_attributes>, [">= 0"])
     end
   else
     s.add_dependency(%q<rest-client>, ["~> 1.6.7"])
@@ -95,10 +131,18 @@ Gem::Specification.new do |s|
     s.add_dependency(%q<json>, ["~> 1.8.0"])
     s.add_dependency(%q<highline>, ["~> 1.6.19"])
     s.add_dependency(%q<fluentd>, ["= 0.10.35"])
+    s.add_dependency(%q<ruby-binlog>, [">= 1.0.0"])
+    s.add_dependency(%q<fluent-plugin-mysql-binlog>, ["~> 0.0.2"])
+    s.add_dependency(%q<mysql2>, ["~> 0.3.11"])
     s.add_dependency(%q<bundler>, [">= 0"])
     s.add_dependency(%q<jeweler>, [">= 0"])
     s.add_dependency(%q<rspec>, [">= 0"])
+    s.add_dependency(%q<timecop>, [">= 0"])
     s.add_dependency(%q<sqlite3>, [">= 0"])
+    s.add_dependency(%q<ruby-prof>, [">= 0"])
+    s.add_dependency(%q<activemodel>, ["~> 4.0.0"])
+    s.add_dependency(%q<activerecord>, ["~> 4.0.0"])
+    s.add_dependency(%q<protected_attributes>, [">= 0"])
   end
 end
 

data/lib/flydata.rb

@@ -12,7 +12,9 @@ lib_dir = File.expand_path(File.dirname(__FILE__))
 ActiveSupport::Dependencies.autoload_paths << lib_dir
 
 module Flydata
+  FLYDATA_DEBUG = !!(ENV['FLYDATA_DEBUG'])
   FLYDATA_HOME = ENV['FLYDATA_HOME'] || "#{ENV['HOME']}/.flydata"
-
+  FLYDATA_GEM_HOME = File.expand_path(File.join(File.dirname(__FILE__), '..'))
+  FLYDATA_TMPL_DIR = File.join(FLYDATA_GEM_HOME, 'tmpl')
   include Flydata::Heroku
 end

data/lib/flydata/api/data_entry.rb

@@ -6,6 +6,10 @@ module Flydata
         @url_path = "/data_ports/:data_port_id/#{@model_name.pluralize}"
         super
       end
+
+      def buffer_stat(data_entry_id, mode = nil)
+        @client.get("/#{@model_name.pluralize}/#{data_entry_id}/buffer_stat/#{mode}")
+      end
     end
   end
 end

data/lib/flydata/api/redshift_cluster.rb

@@ -0,0 +1,15 @@
+module Flydata
+  module Api
+    class RedshiftCluster < Base
+      def initialize(api_client)
+        @model_name = 'redshift_cluster'
+        @url_path = "/#{@model_name.pluralize}"
+        super
+      end
+
+      def run_query(sql)
+        @client.post("#{@url_path}/query", nil, {redshift_query: {body: sql}})
+      end
+    end
+  end
+end

data/lib/flydata/cli.rb

@@ -26,6 +26,7 @@ module Flydata
         puts
         puts
         print_usage
+        raise e if FLYDATA_DEBUG
       end
     end
 

data/lib/flydata/command/base.rb

@@ -8,10 +8,16 @@ module Flydata
 
       def retrieve_data_entries
         data_entries = flydata.get('/data_entries')
-        unless flydata.response.code == 200
+        unless flydata.response.code == 200 and data_entries
           raise "Failed to retrieve data_entries"
         end
-        data_entries
+        data_entries.collect do |de|
+          if Flydata::Preference::DataEntryPreference.conf_exists?(de)
+            Flydata::Preference::DataEntryPreference.load_conf(de)
+          else
+            de
+          end
+        end
       end
       def register_crontab
         data_entries = retrieve_data_entries

data/lib/flydata/command/conf.rb

@@ -0,0 +1,48 @@
+module Flydata
+  module Command
+    class Conf < Base
+      def run
+        data_entries = retrieve_data_entries
+        copy_conf_tmpl(data_entries)
+        usage(data_entries)
+      end
+
+      def copy_templates
+        copy_conf_tmpl(retrieve_data_entries)
+      end
+
+      def help
+        usage(retrieve_data_entries)
+      end
+
+      private
+
+      def copy_conf_tmpl(data_entries)
+        data_entries.each { |de| Flydata::Preference::DataEntryPreference.copy_template(de) }
+      end
+
+      def usage(data_entries)
+        puts "Data Entry Configuration"
+        puts "=" * 64
+        cnt = 0
+        data_entries.each do |de|
+          name = "#{de['display_name']}" || de['name']
+          file = if Flydata::Preference::DataEntryPreference.configurable?(de)
+                   cnt += 1
+                   Flydata::Preference::DataEntryPreference.conf_path(de)
+                 else
+                   "(not supported)"
+                 end
+          puts "#{name.ljust(24)} -> #{file}"
+        end
+        puts "=" * 64
+        puts
+        if cnt < 1
+          puts "There are no configurable data entries."
+        else
+          puts "Edit above files to change the configuration."
+        end
+      end
+    end
+  end
+end

data/lib/flydata/command/encrypt.rb

@@ -0,0 +1,18 @@
+module Flydata
+  module Command
+    class Encrypt < Base
+      def run
+        dp = flydata.data_port.get
+        key = dp['key']
+        text = ask_password
+        encrypted_text = Flydata::Util::Encryptor.encrypt(text, key)
+        puts "Encrypted text: #{encrypted_text}"
+      end
+
+      private
+      def ask_password
+        ask("Enter text: ") { |x| x.echo = "*" }
+      end
+    end
+  end
+end