api-harvester 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c5b682cb063b69e48cbfeede8e43532baa68e215
-  data.tar.gz: 0ce1ac7b04883ced29df0f87ab33120082de583b
+  metadata.gz: dc8ef7a9dc5793b2fd06186d813b03a2d59823e4
+  data.tar.gz: 74a6f1fedc4980a92e5c21c05321b76c75a70e2d
 SHA512:
-  metadata.gz: 6152657838bec26fe2dba1282f18084ef881da2979e32df86c7961f456687cc0982b53c653a8af30e7128908f557302005f02f5932b0c26867f5a888c2bcda2b
-  data.tar.gz: 6ac7666c38611eb685c2ad10d6055020f1b61f2779e55a4af3d40f1a2a9537c5845d310f246d73c7466e1d2eb79d90697f84b711a3f7d321c1a4b76e756b5630
+  metadata.gz: 7fb3e490ca07a6f5aa8183b9a0e33ee8b3248c0b3377ff064a60a603552b56c2a0fb61a293db1d4de9b3bf0bb5235e0b483abac5a8287fad4fd9cd92ed3aa4d4
+  data.tar.gz: 06d77bd4b468a0f7a12fdb8d1b3a8c144d084075aaf2a02bb90ec6581989237ce39e337874aeb0c6d49c9f3cce5a0223604b817156c100bc5fd4080b80a0c39b

data/.document

@@ -0,0 +1,5 @@
+lib/**/*.rb
+bin/*
+- 
+features/**/*.feature
+LICENSE.txt

data/.gitignore

@@ -0,0 +1,50 @@
+# rcov generated
+coverage
+coverage.data
+
+# rdoc generated
+rdoc
+
+# yard generated
+doc
+.yardoc
+
+# bundler
+.bundle
+vendor
+# jeweler generated
+pkg
+
+harvester.yaml
+# Have editor/IDE/OS specific files you need to ignore? Consider using a global gitignore: 
+#
+# * Create a file at ~/.gitignore
+# * Include files you want ignored
+# * Run: git config --global core.excludesfile ~/.gitignore
+#
+# After doing this, these files will be ignored in all your git projects,
+# saving you from having to 'pollute' every project you touch with them
+#
+# Not sure what to needs to be ignored for particular editors/OSes? Here's some ideas to get you started. (Remember, remove the leading # of the line)
+#
+# For MacOS:
+#
+.DS_Store
+
+# For TextMate
+#*.tmproj
+#tmtags
+
+# For emacs:
+#*~
+#\#*
+#.\#*
+
+# For vim:
+#*.swp
+
+# For redcar:
+#.redcar
+
+# For rubinius:
+#*.rbc

data/Gemfile

@@ -0,0 +1,28 @@
+source 'http://rubygems.org'
+group :development do
+  gem "shoulda", ">= 0"
+  gem "rdoc", "~> 3.12"
+  gem "bundler", "~> 1.0"
+  gem "jeweler", "~> 2.0.1"
+  gem "simplecov", ">= 0"
+  gem 'pry'
+  gem 'pry-nav'
+end
+
+
+gem 'faraday'
+gem 'excon'
+gem 'commander'
+gem 'ezmq'
+gem 'msgpack'
+
+
+gem 'influxdb', '~> 0.1.9'
+gem 'eye'
+gem 'moneta'
+gem 'sinatra'
+gem 'rainbow'
+gem 'oj'
+
+gem 'terminal-announce'
+gem 'settingslogic'

data/Gemfile.lock

@@ -0,0 +1,154 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    activesupport (4.2.3)
+      i18n (~> 0.7)
+      json (~> 1.7, >= 1.7.7)
+      minitest (~> 5.1)
+      thread_safe (~> 0.3, >= 0.3.4)
+      tzinfo (~> 1.1)
+    addressable (2.3.8)
+    builder (3.2.2)
+    cause (0.1)
+    celluloid (0.15.2)
+      timers (~> 1.1.0)
+    celluloid-io (0.15.0)
+      celluloid (>= 0.15.0)
+      nio4r (>= 0.5.0)
+    coderay (1.1.0)
+    commander (4.3.4)
+      highline (~> 1.7.2)
+    contracts (0.11.0)
+    descendants_tracker (0.0.4)
+      thread_safe (~> 0.3, >= 0.3.1)
+    docile (1.1.5)
+    excon (0.45.4)
+    eye (0.6.4)
+      celluloid (~> 0.15.0)
+      celluloid-io (~> 0.15.0)
+      sigar (~> 0.7.2)
+      state_machine
+      thor
+    ezmq (0.4.4)
+      ffi-rzmq (~> 2.0)
+    faraday (0.9.1)
+      multipart-post (>= 1.2, < 3)
+    ffi (1.9.10)
+    ffi-rzmq (2.0.4)
+      ffi-rzmq-core (>= 1.0.1)
+    ffi-rzmq-core (1.0.3)
+      ffi (~> 1.9)
+    git (1.2.9.1)
+    github_api (0.12.4)
+      addressable (~> 2.3)
+      descendants_tracker (~> 0.0.4)
+      faraday (~> 0.8, < 0.10)
+      hashie (>= 3.4)
+      multi_json (>= 1.7.5, < 2.0)
+      nokogiri (~> 1.6.6)
+      oauth2
+    hashie (3.4.2)
+    highline (1.7.2)
+    i18n (0.7.0)
+    influxdb (0.1.9)
+      cause
+      json
+    jeweler (2.0.1)
+      builder
+      bundler (>= 1.0)
+      git (>= 1.2.5)
+      github_api
+      highline (>= 1.6.15)
+      nokogiri (>= 1.5.10)
+      rake
+      rdoc
+    json (1.8.3)
+    jwt (1.5.1)
+    method_source (0.8.2)
+    mini_portile (0.6.2)
+    minitest (5.8.0)
+    moneta (0.8.0)
+    msgpack (0.6.2)
+    multi_json (1.11.2)
+    multi_xml (0.5.5)
+    multipart-post (2.0.0)
+    nio4r (1.1.1)
+    nokogiri (1.6.6.2)
+      mini_portile (~> 0.6.0)
+    oauth2 (1.0.0)
+      faraday (>= 0.8, < 0.10)
+      jwt (~> 1.0)
+      multi_json (~> 1.3)
+      multi_xml (~> 0.5)
+      rack (~> 1.2)
+    oj (2.12.10)
+    pry (0.10.1)
+      coderay (~> 1.1.0)
+      method_source (~> 0.8.1)
+      slop (~> 3.4)
+    pry-nav (0.2.4)
+      pry (>= 0.9.10, < 0.11.0)
+    rack (1.6.4)
+    rack-protection (1.5.3)
+      rack
+    rainbow (2.0.0)
+    rake (10.4.2)
+    rdoc (3.12.2)
+      json (~> 1.4)
+    settingslogic (2.0.9)
+    shoulda (3.5.0)
+      shoulda-context (~> 1.0, >= 1.0.1)
+      shoulda-matchers (>= 1.4.1, < 3.0)
+    shoulda-context (1.2.1)
+    shoulda-matchers (2.8.0)
+      activesupport (>= 3.0.0)
+    sigar (0.7.3)
+    simplecov (0.10.0)
+      docile (~> 1.1.0)
+      json (~> 1.8)
+      simplecov-html (~> 0.10.0)
+    simplecov-html (0.10.0)
+    sinatra (1.4.6)
+      rack (~> 1.4)
+      rack-protection (~> 1.4)
+      tilt (>= 1.3, < 3)
+    slop (3.6.0)
+    state_machine (1.2.0)
+    terminal-announce (1.0.0)
+      bundler
+      contracts
+      rainbow
+    thor (0.19.1)
+    thread_safe (0.3.5)
+    tilt (2.0.1)
+    timers (1.1.0)
+    tzinfo (1.2.2)
+      thread_safe (~> 0.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.0)
+  commander
+  excon
+  eye
+  ezmq
+  faraday
+  influxdb (~> 0.1.9)
+  jeweler (~> 2.0.1)
+  moneta
+  msgpack
+  oj
+  pry
+  pry-nav
+  rainbow
+  rdoc (~> 3.12)
+  settingslogic
+  shoulda
+  simplecov
+  sinatra
+  terminal-announce
+
+BUNDLED WITH
+   1.10.4

data/README.md

@@ -0,0 +1,2 @@
+harvest key_transactions --from https://api.newrelic.com/v2/key_transactions.json --every 5s
+harvester harvest key_transactions --from https://api.newrelic.com/v2/key_transactions.json --every 5

data/Rakefile

@@ -0,0 +1,51 @@
+# encoding: utf-8
+
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://guides.rubygems.org/specification-reference/ for more options
+  gem.name = "harvester"
+  gem.homepage = "http://github.com/kkwoker/harvester"
+  gem.license = "MIT"
+  gem.summary = %Q{TODO: one-line summary of your gem}
+  gem.description = %Q{TODO: longer description of your gem}
+  gem.email = "kkwoker@gmail.com"
+  gem.authors = ["kkwoker"]
+  # dependencies defined in Gemfile
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+desc "Code coverage detail"
+task :simplecov do
+  ENV['COVERAGE'] = "true"
+  Rake::Task['test'].execute
+end
+
+task :default => :test
+
+require 'rdoc/task'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "harvester #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/bin/harvester

@@ -0,0 +1,37 @@
+#!/usr/bin/env ruby
+
+require 'commander'
+require_relative '../lib/harvester'
+
+Commander.configure do
+  program :name, 'Harvester'
+  program :version, '0.0.1'
+  program :description, 'harvest from endpoints'
+  program :help, 'Author', 'Kinnan Kwok <kkwoker@gmail.com>'
+
+  default_command :harvest
+
+  command :harvest do |command|
+    command.syntax = 'harvester harvest [options]'
+    command.summary = ''
+    command.description = 'Starts processes for harvesting data'
+    command.example 'description', 'command example'
+    command.option '--from STRING', String, 'Adds an endpoint to harvest from'
+    command.option '--with STRING', String, 'Adds a parsing method ruby file'
+    command.option '--every INTEGER', Integer, 'Adds an interval for polling'
+    command.action do |_args, options|
+      # Do something or c.when_called Harvester::Commands::Harvest
+      abort 'Must provide parameters --from, --with, and --every' unless options.from && options.with && options.every
+      name = options.with
+      endpoint = options.from
+      timeout = options.every
+
+      # Create an eye file.
+      EyeGenerator.initialize name, endpoint, timeout
+      # Load Eye file
+      system "eye load #{name}.d/config.eye"
+      system "eye restart #{name}"
+      
+    end
+  end
+end

data/harvester.gemspec

@@ -0,0 +1,43 @@
+Gem::Specification.new do |gem|
+  gem.name        = 'api-harvester'
+  gem.version     = '0.0.2'
+  gem.licenses    = 'MIT'
+  gem.authors     = ['Kinnan Kwok']
+  gem.email       = 'kkwoker@gmail.com'
+  gem.summary     = 'Allows you to specify an endpoint to retrieve data from, parse it, and store into any database'
+  gem.require_paths = ["lib"]
+  # gem.executables   = ["harvester"]
+  # gem.files         = [
+  #   ".document",
+  #   ".gitignore",
+  #   "Gemfile",
+  #   "Gemfile.lock",
+  #   "README.md",
+  #   "Rakefile",
+  #   "bin/harvester",
+  #   "harvester.gemspec",
+  #   "lib/harvester.rb",
+  #   "lib/harvester/eye_generator.rb"
+  #   "lib/harvester/parser.rb"
+  #   "lib/harvester/pusher.rb"
+  #   "lib/harvester/puller.rb"
+  #   "lib/harvester/settings.rb"
+  # ]
+
+  gem.homepage = "http://github.com/kkwoker/api-harvester"
+  gem.add_runtime_dependency 'ezmq', '~> 0.4.4'
+  gem.add_runtime_dependency 'faraday'
+  gem.add_runtime_dependency 'excon'
+  gem.add_runtime_dependency 'commander'
+  gem.add_runtime_dependency 'msgpack'
+  gem.add_runtime_dependency 'influxdb', '~> 0.1.9'
+  gem.add_runtime_dependency 'eye'
+  gem.add_runtime_dependency 'moneta'
+  gem.add_runtime_dependency 'sinatra'
+  gem.add_runtime_dependency 'rainbow'
+  gem.add_runtime_dependency 'oj'
+  gem.add_runtime_dependency 'terminal-announce'
+  gem.add_runtime_dependency 'settingslogic'
+
+  gem.add_development_dependency 'pry'
+end

data/lib/harvester/eye_generator.rb

@@ -0,0 +1,33 @@
+require_relative '../../template/default_eye'
+
+module EyeGenerator
+  def self.create_eye_config name
+    config = Default.eye_config name
+    File.open("#{name}.d/config.eye", 'w') { |file| file.write(config)}
+  end
+
+  def self.create_puller name, endpoint, timeout
+    puller = Default.puller name, endpoint, timeout
+    File.open("#{name}.d/puller.rb", 'w') { |file| file.write(puller)}
+  end
+
+  def self.create_parser name, endpoint
+    parser = Default.parser name, endpoint
+    File.open("#{name}.d/parser.rb", 'w') { |file| file.write(parser)}
+  end
+
+  def self.create_pusher name, endpoint
+    pusher = Default.pusher name, endpoint
+    File.open("#{name}.d/pusher.rb", 'w') { |file| file.write(pusher)}
+  end
+
+  def self.initialize name, endpoint, timeout
+    system "mkdir -p #{name}.d"
+    system "mkdir -p #{name}.d/logs"
+    system "mkdir -p #{name}.d/pids"
+    create_eye_config name
+    create_puller name, endpoint, timeout
+    create_parser name, endpoint
+    create_pusher name, endpoint
+  end
+end

data/lib/harvester/parser.rb

@@ -0,0 +1,23 @@
+require 'pry'
+require 'ezmq'
+require 'msgpack'
+require 'oj'
+require_relative 'settings'
+require_relative '../../key_transactions'
+module Parser
+  @broadcaster = EZMQ::Publisher.new :bind, port: 6000, encode: -> m { Oj.dump m }
+  def self.activate topic, endpoint
+    puts topic, endpoint
+    listener = EZMQ::Subscriber.new :connect, port: 5000, topic: topic, decode: -> m { Oj.load m }
+    listener.listen do |message, topic|
+      data = plow message, topic
+      data.each do |item|
+        puts @broadcaster.send item, topic: topic
+      end
+    end
+  end
+
+  def self.plow message, parse
+    send(parse, message)
+  end
+end

data/lib/harvester/puller.rb

@@ -0,0 +1,32 @@
+require 'ezmq'
+require 'msgpack'
+require 'faraday'
+require 'oj'
+require 'excon'
+require_relative 'settings'
+
+module Puller
+  @broadcaster = EZMQ::Publisher.new :bind, port: 5000
+  def self.fetch_data(endpoint)
+    conn = Faraday.new(url: endpoint) do |faraday|
+      faraday.adapter :excon
+      faraday.headers['X-Api-Key'] = Settings.newrelic_key
+    end
+    conn.get.body
+  # rescue Exception => e
+  #   puts "Puller#fetch_data"
+  #   puts "#{e.inspect}: #{e.message}"
+  end
+
+  def self.send(data, topic)
+    # puts "Puller#send"
+    @broadcaster.send data, topic: topic
+  end
+
+  def self.activate(name, endpoint, timeout)
+    loop do
+      puts send fetch_data(endpoint), name
+      sleep timeout
+    end
+  end
+end

data/lib/harvester/pusher.rb

@@ -0,0 +1,33 @@
+require 'pry'
+require 'ezmq'
+require 'oj'
+require 'influxdb'
+require_relative 'settings'
+
+module Pusher
+  @connection = InfluxDB::Client.new host: Settings.influx_host, username: Settings.influx_user, password: Settings.influx_pass
+
+  def self.activate topic, endpoint
+    listener = EZMQ::Subscriber.new :connect, port: 6000, topic: topic, decode: -> m { Oj.load m }
+    listener.listen do |message, topic|
+      puts write message
+    end
+  end
+
+  def self.write data
+    series = data.delete :series_name
+    db = data.delete :database_name
+    use_or_create db
+    @connection.write_point(series, data)
+    data.length
+  end
+
+  def self.use_or_create(db_name)
+    dbs = @connection.get_database_list.map { |db| db["name"] }
+    @connection.create_database db_name unless dbs.include? db_name
+    @connection = InfluxDB::Client.new db_name, host: Settings.influx_host, 
+                                              username: Settings.influx_user, 
+                                              password: Settings.influx_pass
+  end
+
+end

data/lib/harvester/settings.rb

@@ -0,0 +1,15 @@
+require 'settingslogic'
+require 'terminal-announce'
+class Settings < Settingslogic
+	config_paths = %w(/etc /usr/local/etc ~/.config . ../)
+
+	config_paths.each do |config_path|
+		config_file = File.expand_path "#{ config_path }/harvester.yaml"
+		source config_file if File.exist? config_file
+	end
+
+	load!
+rescue Errno::ENOENT
+	Announce.failure "Unable to locate configuration in #{ config_paths }."
+	abort
+end

data/lib/harvester.rb

@@ -0,0 +1,4 @@
+require_relative 'harvester/puller'
+require_relative 'harvester/parser'
+require_relative 'harvester/pusher'
+require_relative 'harvester/eye_generator'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: api-harvester
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - Kinnan Kwok
@@ -208,10 +208,25 @@ dependencies:
         version: '0'
 description: 
 email: kkwoker@gmail.com
-executables: []
+executables:
+- harvester
 extensions: []
 extra_rdoc_files: []
-files: []
+files:
+- ".document"
+- ".gitignore"
+- Gemfile
+- Gemfile.lock
+- README.md
+- Rakefile
+- bin/harvester
+- harvester.gemspec
+- lib/harvester.rb
+- lib/harvester/eye_generator.rb
+- lib/harvester/parser.rb
+- lib/harvester/puller.rb
+- lib/harvester/pusher.rb
+- lib/harvester/settings.rb
 homepage: http://github.com/kkwoker/api-harvester
 licenses:
 - MIT