flydata 0.0.1.nc1

data/lib/flydata/cron.rb

@@ -0,0 +1,88 @@
+module Flydata
+  class Cron
+    FLYDATA_HOME=Flydata::HOME_DIR
+    CRON_TMP_PATH="#{FLYDATA_HOME}/cron_list.tmp"
+    CRON_BACKUP_PATH="#{FLYDATA_HOME}/cron_list.back"
+    LOG_PATH="#{FLYDATA_HOME}/flydata_cron.log"
+    ROUTINE_BASE_COMMAND="#{FLYDATA_HOME}/bin/flydata routine >> #{LOG_PATH} 2>&1"
+    if ENV['FLYDATA_API_HOST'] and ENV['FLYDATA_API_HOST'] != ""
+      ROUTINE_COMMAND="export FLYDATA_API_HOST=#{ENV['FLYDATA_API_HOST']} && #{ROUTINE_BASE_COMMAND}"
+    else
+      ROUTINE_COMMAND=ROUTINE_BASE_COMMAND
+    end
+
+    CRON_LINE="15 4 * * * #{ROUTINE_COMMAND}"
+    #CRON_LINE="* * * * * #{ROUTINE_COMMAND}"
+    REMOVE_OPTION="-r"
+
+    def initialize
+    end
+    def update(option=nil)
+      arg = option ? option : ''
+
+      # Check condition
+      check_crontab_command
+      dump_cron_setting
+      reagistered = check_cron_setting
+
+      # Remove mode
+      if arg == REMOVE_OPTION
+        unless reagistered
+          puts "Skip removing. There are no flydata jobs in crontab."
+        else
+          remove_cron_setting
+          puts "Updated crontab to remove the deletion of your log data."
+        end
+      # Add mode
+      else
+        if reagistered
+          puts "Skip register. Already cron setup done."
+        else
+          add_cron_setting
+          puts "Updated crontab to manage the deletion of your log data."
+        end
+      end
+      true
+    end
+
+    private
+    def check_crontab_command
+      raise "Failed to update. Crontab is not installed." unless system("which crontab > /dev/null")
+    end
+    def dump_cron_setting
+      system("crontab -l > #{CRON_BACKUP_PATH}")
+    end
+    def check_cron_setting
+      already_exist = false
+      File.open(CRON_BACKUP_PATH, 'r') { |file|
+        while l = file.gets
+          if l =~ /#{ROUTINE_COMMAND}/
+            already_exist = true
+            break
+          end
+        end
+      }
+      already_exist
+    end
+    def add_cron_setting
+      FileUtils.copy(CRON_BACKUP_PATH, CRON_TMP_PATH)
+      File.open(CRON_TMP_PATH, "a") { |file|
+        file.puts(CRON_LINE)
+      }
+      ret = system("crontab #{CRON_TMP_PATH}")
+      raise "Failed to register flydata routine job to crontab." unless ret
+    end
+    def remove_cron_setting
+      File.open(CRON_TMP_PATH, "w") { |dst|
+        File.open(CRON_BACKUP_PATH, 'r') { |src|
+          while l = src.gets
+            dst.puts(l) unless l =~ /#{ROUTINE_COMMAND}/
+          end
+        }
+      }
+      ret = system("crontab #{CRON_TMP_PATH}")
+      raise "Failed to remove flydata routine job from crontab." unless ret
+    end
+
+  end
+end

data/lib/flydata/flydata_crontab.sh

@@ -0,0 +1,116 @@
+#!/bin/bash
+
+# Usage:
+#  flydata_crontab
+
+######## Parameters
+FLYDATA_HOME=~/".flydata"
+CRON_TMP_PATH="$FLYDATA_HOME/cron_list.tmp"
+CRON_BACKUP_PATH="$FLYDATA_HOME/cron_list.back"
+LOG_PATH="$FLYDATA_HOME/flydata_cron.log"
+ROUTINE_COMMAND="$FLYDATA_HOME/bin/flydata routine >> $LOG_PATH 2>&1"
+if [[ "$FLYDATA_API_HOST" != "" ]]; then
+  ROUTINE_COMMAND="export FLYDATA_API_HOST=$FLYDATA_API_HOST && $ROUTINE_COMMAND"
+fi
+#CRON_LINE='15 4 * * *'" $ROUTINE_COMMAND"
+CRON_LINE='* * * * *'" $ROUTINE_COMMAND"
+
+ARG=$1
+REMOVE_OPTION="-r"
+CRON_SETTING_DONE=0 # flag
+
+
+######## Functions
+check_flydata_setup_done()
+{
+  which flydata > /dev/null
+  if [ "$?" -ne "0" ]; then
+    echo "flydata command not found. Flydata setup has not been done, yet."
+    exit -1
+  fi
+  if [ ! -w $FLYDATA_HOME ]; then
+    echo "Flydata setup has not been done, yet. Run setup command."
+    echo "  -> flydata setup"
+    exit -1
+  fi
+}
+
+check_crontab_commnad()
+{
+  which crontab > /dev/null
+  if [ "$?" -ne "0" ]; then
+    echo "No crontab command. Please install cron."
+    exit -2
+  fi
+}
+
+check_crond_process()
+{
+  crond_exist=`ps aux | grep crond | grep -v grep | wc -l`
+  echo $crond_exist
+  if [ "$crond_exist" -ne "1" ]; then
+    echo "No crond process. Please start crond process."
+    exit -3
+  fi
+}
+
+dump_cron_setting()
+{
+  crontab -l > $CRON_BACKUP_PATH
+}
+
+check_cron_setting()
+{
+  CRON_SETTING_DONE=`cat $CRON_BACKUP_PATH | grep "flydata routine" | wc -l`
+  if [ "$?" -ne "0" ]; then
+    echo "Failed to check cron setting file."
+    exit -5
+  fi
+}
+
+add_cron_setting()
+{
+  cat $CRON_BACKUP_PATH > $CRON_TMP_PATH
+  echo "$CRON_LINE" >> $CRON_TMP_PATH
+  crontab $CRON_TMP_PATH
+  if [ "$?" -ne "0" ]; then
+    echo "Failed to register flydata routine job to crontab."
+    exit -6
+  fi
+}
+
+remove_cron_setting()
+{
+  cat "$CRON_BACKUP_PATH" | grep -v "flydata routine" > $CRON_TMP_PATH
+  crontab $CRON_TMP_PATH
+  if [ "$?" -ne "0" ]; then
+    echo "Failed to remove flydata routine job from crontab."
+    exit -7
+  fi
+}
+
+
+######## Main
+ARG="$1"
+check_flydata_setup_done
+check_crontab_commnad
+# check_crond_process
+dump_cron_setting
+check_cron_setting
+
+# Skip If add mode and registered
+if [[ "$ARG" != "$REMOVE_OPTION" && $CRON_SETTING_DONE -gt 0 ]]; then
+  echo "Skip. Already cron setup done."
+  exit 0
+fi
+
+# Remove mode -> remove setting
+if [ "$ARG" == "$REMOVE_OPTION" ]; then
+  remove_cron_setting
+  exit 0
+fi
+
+# Add mode -> add setting
+add_cron_setting
+echo "Updated crontab to manage the deletion of your log data."
+exit 0

data/lib/flydata/helpers.rb

@@ -0,0 +1,20 @@
+module Flydata
+  module Helpers
+    module_function
+    def parse_command(cmd)
+      klass = Flydata::Command::Base
+      method = cmd
+      if cmd.include?(':')
+        class_name, method = cmd.split(':')
+        klass = to_command_class(class_name)
+      end
+      [klass, method]
+    end
+    def to_command_class(class_name)
+      eval("Flydata::Command::#{class_name.camelcase}")
+    end
+    def home_directory
+      ENV['HOME'] || Dir.pwd
+    end
+  end
+end

data/lib/flydata/log_monitor.rb

@@ -0,0 +1,93 @@
+module Flydata
+  # Directory structure
+  #  log_dir
+  #    - log_path
+  #    + .flydata
+  #      + [log_file_name]
+  #        + trash  <- temprary directory
+  #          - [log_archived_files]
+  #        + backup <- backup archive files when initialize
+  #          - [log_archived_files]
+  class LogMonitor
+    FLYDATA_DIR = '.flydata'
+    TRASH_DIR = 'trash'
+    BACKUP_DIR = 'backup'
+    DEFAULT_DAYS_TO_TRASH = 7 #days
+    DEFAULT_DAYS_TO_DELETE = DEFAULT_DAYS_TO_TRASH + 7 #days
+
+    def initialize(log_path, options={})
+      # check arguments
+      raise "Cannot access to log path... #{log_path}" unless File.exist?(log_path)
+      @log_path = log_path
+      @options = options
+      @days_to_trash = @options[:days_to_trash] ?
+        @options[:days_to_trash] : DEFAULT_DAYS_TO_TRASH
+      @days_to_delete = @options[:days_to_delete] ?
+        @options[:days_to_delete] : DEFAULT_DAYS_TO_DELETE
+      @setup_flag = false
+      # initialize paths
+      @flydata_path = File.join(File.dirname(@log_path),
+                                FLYDATA_DIR,
+                                File.basename(@log_path) + ".d")
+      @backup_path = File.join(@flydata_path, BACKUP_DIR)
+      @trash_path = File.join(@flydata_path, TRASH_DIR)
+    end
+    def setup
+      # create directories
+      FileUtils.mkdir_p(@flydata_path)
+      FileUtils.mkdir_p(@backup_path)
+      # backup_files to avoid to delete non-uploaded files
+      backup_files
+      @setup_flag = true
+      self
+    end
+    def rotate
+      check_setup_done
+      # move files to trash
+      trash_files
+      # delete trashed files
+      delete_files
+    end
+
+    private
+    def check_setup_done
+      raise "Setup hasn't been done, yet." unless @setup_flag
+    end
+    def archived_files
+      Dir::glob("#{@log_path}?*")
+    end
+    def trashed_files
+      Dir::glob("#{File.join(@trash_path, File.basename(@log_path))}?*")
+    end
+    def backup_files
+      # use trash dir as checkpoint to backup initial archive files
+      unless File.exist?(@trash_path)
+        archived_files.each {|f|
+          puts "buckup file: #{f} -> #{@backup_path}"
+          FileUtils.mv(f, @backup_path)
+        }
+        FileUtils.mkdir_p(@trash_path)
+      end
+    end
+    def trash_files
+      archived_files.each {|f|
+        # check date and trash
+        if @days_to_trash.days.ago > File.stat(f).mtime
+          to_path = File.join(@trash_path, File.basename(f))
+          to_path = "#{to_path}_#{Time.now.strftime('%Y%m%d_%H%M%S')}" if File.exist?(to_path)
+          puts "trash file: #{f} -> #{to_path}"
+          FileUtils.mv(f, File.join(@trash_path, File.basename(f)))
+        end
+      }
+    end
+    def delete_files
+      trashed_files.each {|f|
+        if @days_to_delete.days.ago > File.stat(f).mtime
+          file_path = File.join(@trash_path, File.basename(f))
+          puts "delete file: #{file_path}"
+          FileUtils.rm(file_path)
+        end
+      }
+    end
+  end
+end

data/lib/flydata/proxy.rb

@@ -0,0 +1,16 @@
+module Hapyrus
+  class Proxy
+    attr_accessor :command_class
+    def run_command(method, *args)
+      args.flatten!
+      if args.size > 0
+        @command_class.new.send(method, *args)
+      else
+        @command_class.new.send(method)
+      end
+    end
+    def method_missing(method, *args)
+      run_command(method, args)
+    end
+  end
+end

data/spec/flydata/api/data_entry_spec.rb

@@ -0,0 +1,17 @@
+require 'spec_helper'
+
+module Flydata
+  module Api
+    describe DataEntry do
+      describe '#create' do
+        it 'creates data entry' do
+          api_client = double('api_client')
+          api_client.stub(:post)
+
+          api = DataEntry.new(api_client)
+          api.create(data_port_id: 1, log_path: 'log-path')
+        end
+      end
+    end
+  end
+end

data/spec/flydata/command/sender_spec.rb

@@ -0,0 +1,37 @@
+require 'spec_helper'
+
+module Flydata
+  module Command
+    describe Sender do
+      before :each do
+        @sender = Sender.new
+      end
+      describe '#start' do
+        it 'starts fluentd' do
+          api_client = double('api_client')
+          api_client.stub_chain(:data_port, :get, :[]).and_return 'active'
+          ApiClient.stub(:new).and_return api_client
+
+          @sender.start
+        end
+      end
+      describe '#stop' do
+        it 'stops fluentd' do
+          @sender.stop
+        end
+      end
+      describe '#restart' do
+        before :all do
+          @sender = Sender.new
+          @sender.start
+        end
+        it 'restart fluentd' do
+          @sender.restart
+        end
+        after :all do
+          @sender.stop
+        end
+      end
+    end
+  end
+end

data/spec/flydata_spec.rb

@@ -0,0 +1,4 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+describe "Flydata" do
+end

data/spec/spec_helper.rb

@@ -0,0 +1,12 @@
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'rspec'
+require 'flydata'
+
+# Requires supporting files with custom matchers and macros, etc,
+# in ./support/ and its subdirectories.
+Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each {|f| require f}
+
+RSpec.configure do |config|
+  
+end

metadata

@@ -0,0 +1,291 @@
+--- !ruby/object:Gem::Specification
+name: flydata
+version: !ruby/object:Gem::Version
+  version: 0.0.1.nc1
+  prerelease: 6
+platform: ruby
+authors:
+- Koichi Fujikawa
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-10-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rest-client
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: i18n
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: highline
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: jeweler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: autotest
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: autotest-standalone
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: autotest-notification
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: ZenTest
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: FlyData Command Line Interface
+email: sysadmin@flydata.co
+executables:
+- flydata
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rspec
+- Gemfile
+- Gemfile.lock
+- Rakefile
+- VERSION
+- bin/flydata
+- flydata.gemspec
+- lib/flydata.rb
+- lib/flydata/api/base.rb
+- lib/flydata/api/data_entry.rb
+- lib/flydata/api/data_port.rb
+- lib/flydata/api_client.rb
+- lib/flydata/cli.rb
+- lib/flydata/command/base.rb
+- lib/flydata/command/crontab.rb
+- lib/flydata/command/login.rb
+- lib/flydata/command/restart.rb
+- lib/flydata/command/routine.rb
+- lib/flydata/command/sender.rb
+- lib/flydata/command/setlogdel.rb
+- lib/flydata/command/setup.rb
+- lib/flydata/command/start.rb
+- lib/flydata/command/stop.rb
+- lib/flydata/credentials.rb
+- lib/flydata/cron.rb
+- lib/flydata/flydata_crontab.sh
+- lib/flydata/helpers.rb
+- lib/flydata/log_monitor.rb
+- lib/flydata/proxy.rb
+- spec/flydata/api/data_entry_spec.rb
+- spec/flydata/command/sender_spec.rb
+- spec/flydata_spec.rb
+- spec/spec_helper.rb
+homepage: http://flydata.co/
+licenses:
+- All right reserved.
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: 1073163683650031342
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>'
+    - !ruby/object:Gem::Version
+      version: 1.3.1
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: FlyData CLI
+test_files: []