messenger_pigeon 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 9fdb8c895d8bb1bcbe17ca6b03c1c5b9d5817f5a
+  data.tar.gz: 00b928c5bdb4bbdeda14f782c444ebb3c41579b4
+SHA512:
+  metadata.gz: 99d56f47b366c771b0cd3cb1fe5c22943d2807dd26fd863e0a785aa0636fb08f0165b8738374ea5da40a383034422497ff319db63aa1bb5c2aefba1e2a226dd0
+  data.tar.gz: 23e2a014dd88653a1e4f511f841221036fc254ce62e58d612aac538f02b44f3a09551cb4092d4bf862605a5c22a46e13b642bbd61db0b486cb30dafdcdf8efea

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2015 Chris Mann
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,227 @@
+# MessengerPigeon
+
+A means of getting data from one location to another.
+
+The goal of MessengerPigeon is to provide a highly-configurable and adaptable
+animal to take data from any number of sources, and copy that data to any
+number of destinations.  A pigeon may modify, drop or add to your data while
+in transit, but should only do these things when you ask it nicely.
+
+MessengerPigeon is under heavy development.
+
+## Configuration
+Example Configuration demonstrating some features:
+
+* CSV source to an OrgMode target
+* Setting the data format for a target
+* Globbed file selection
+* Source hooks
+* Data Filtering
+* Pre and Post-filter data transformations
+
+~/.messengerpigeonrc:
+```ruby
+{
+  targets: {
+    'OrgMode' => {
+      type: MessengerPigeon::OrgMode,
+      options: {
+        file: 'clocks.org',
+        refile_target: '%{project}',
+        data_format: proc do |d|
+          clock = '  CLOCK: [%{date} %{start_time}]--[%{date} %{end_time}] =>  %{duration}'
+          clock % d
+        end
+      }
+    }
+  },
+  sources: {
+    'TimeTracker' => {
+      type: MessengerPigeon::CSV,
+      options: {
+        file_glob: 'test.csv',
+        on_complete: :archive
+      }
+    }
+  },
+  pigeons: {
+    'Gemima' => {
+      source: 'TimeTracker',
+      filters: {
+         project: "Work"
+      },
+      transforms: {
+        pre_filter: {
+          date: proc { |a| Date.parse(a).strftime('%Y-%m-%d') },
+          duration: proc do |a|
+            r = format('%.2f', a.to_i + (a.to_f - a.to_i) / 100 * 60)
+            r.sub(/\./, ':')
+          end
+        },
+        post_filter: {}
+      },
+      target: 'OrgMode'
+    }
+  }
+}
+```
+
+## Modules
+
+### CSV
+#### Source
+```ruby
+...
+sources: {
+  'TimeTracker' => {
+    type: MessengerPigeon::CSV,
+    options: {
+      file_glob: 'test.csv',
+      on_complete: :archive
+    }
+  }
+}
+...
+```
+#### Target
+Not yet implemented
+
+### Console
+#### Source
+Not yet implemented.
+#### Target
+```ruby
+...
+targets: {
+  'Console => {
+    type: MessengerPigeon::Console,
+    options: {}
+   }
+}
+...
+```
+### OrgMode
+#### Source
+Not yet implemented
+#### Target
+```ruby
+...
+targets: {
+  'OrgMode' => {
+    type: MessengerPigeon::OrgMode,
+    options: {
+      file: 'clocks.org',
+      refile_target: '%{project}',
+      data_format: proc do |d|
+        clock = '  CLOCK: [%{date} %{start_time}]--[%{date} %{end_time}] =>  %{duration}'
+        clock % d
+      end
+    }
+  }
+}
+...
+```
+### Redmine
+Configuration should be self-explanatory except for perhaps the 'resource'
+option.  This is required option that corresponds to a capitalised,
+camel-cased and singular equivalent to the resources listed here:
+
+http://www.redmine.org/projects/redmine/wiki/Rest_api
+
+E.g., 'time_entries' becomes 'TimeEntry'
+#### Source
+'mode' is either ':all', or ':specific'.
+
+Example for :all to get all issues in Project 2.
+```ruby
+...
+sources: {
+  'Redmine' => {
+    type: MessengerPigeon::Redmine,
+    options: {
+      resource: "Issue",
+      site: 'https://url.to.redmine.org',
+      user: 'username',
+      password: 's3kr1t'
+      mode: :all,
+      params: {
+        status_id: '*',
+        project_id: '2'
+      }
+    }
+  }
+},
+...
+```
+
+Example for :specific to get the issue with ID = 30
+```ruby
+...
+sources: {
+  'Redmine' => {
+    type: MessengerPigeon::Redmine,
+    options: {
+      resource: "Issue",
+      site: 'https://url.to.redmine.org',
+      user: 'username',
+      password: 's3kr1t'
+      mode: :specific,
+      key: 30
+    }
+  }
+},
+...
+```
+
+#### Target
+Note: The generators section of the configuration creates keys in the hash.  This
+should give a reasonable overview of what is required.  See the description of
+keys required on the particular resource page for more information.
+
+```ruby
+...
+targets: {
+  'Redmine' => {
+    type: MessengerPigeon::Redmine,
+    options: {
+      resource: "TimeEntry",
+      site: 'https://url.to.redmine.org',
+      user: 'username',
+      password: 's3kr1t'
+    }
+  },
+},
+pigeons: {
+  'Gemima' => {
+    filters: {
+      issue_id: proc { |a| !a.nil? },
+      activity_id: proc { |a| !a.nil? }
+    },
+    generators: {
+      activity_id: proc { |a| a[:activity_id] || '9' },
+      issue_id: proc do |a|
+        if a[:description]
+          m = a[:description].match(/\#(?<id>[0-9]+)/)
+          m[:id]
+        end
+      end,
+      hours: proc { |a| a[:duration] },
+      spent_on: proc { |a| a[:date] },
+      comments: proc do |a|
+        a[:description].sub(/\#[0-9]+ */, '') if a[:description]
+      end
+    },
+    transforms: {
+      pre_filter: {
+        date: proc { |a| Date.parse(a).strftime('%Y-%m-%d') }
+      },
+      post_filter: {}
+    },
+    source: 'MySource',
+    target: 'Redmine'
+  }
+}
+...
+```
+### And more?
+Suggestions / contributions of more modules are welcome

data/bin/messenger-pigeon

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+require 'messenger_pigeon'
+
+opts = MessengerPigeon::CLI.parse_options
+MessengerPigeon.flock(opts)

data/lib/messenger_pigeon/cli.rb

@@ -0,0 +1,20 @@
+module MessengerPigeon
+  module CLI
+    module_function
+
+    def parse_options
+      options = {
+        config: File.expand_path(ENV['MESSENGER_PIGEON_RC'] ||
+                                 '~/.messenger-pigeon.rc')
+      }
+      OptionParser.new do |opts|
+        opts.banner = "Usage: #{$PROGRAM_NAME} [options]"
+        opts.on('-c', '--config file', 'Configuration file') do |f|
+          options[:config] = f
+        end
+      end.parse!
+      Config.pigeons = ARGV
+      options
+    end
+  end
+end

data/lib/messenger_pigeon/config.rb

@@ -0,0 +1,14 @@
+module MessengerPigeon
+  # MessengerPigeon Configuration
+  module Config
+    extend self
+
+    attr_reader :conf
+    attr_accessor :pigeons
+
+    def load(config)
+      @conf ||= {}
+      @conf.merge!(binding.eval(File.read(config)))
+    end
+  end
+end

data/lib/messenger_pigeon/console.rb

@@ -0,0 +1,25 @@
+
+module MessengerPigeon
+  # Console source/target module
+  module Console
+    # Source definition
+    class Source
+      def initialize
+        fail 'Not Implemented'
+      end
+    end
+
+    # Target definition
+    class Target
+      def initialize(_options)
+      end
+
+      def update(data)
+        puts data
+      end
+
+      def write
+      end
+    end
+  end
+end

data/lib/messenger_pigeon/csv.rb

@@ -0,0 +1,52 @@
+require 'csv'
+CSVLib = CSV
+module MessengerPigeon
+  # CSV source/target module
+  module CSV
+    # Source definition
+    class Source
+      def initialize(options)
+        # :file_glob
+        @files = Dir.glob(options[:file_glob])
+        @on_complete = options[:on_complete]
+      end
+
+      def read
+        @data = @files.collect { |f| read_file f }.flatten
+      end
+
+      def complete
+        case @on_complete
+        when :archive
+          archive_files
+        end
+      end
+
+      private
+
+      def archive_files
+        @files.each do |f|
+          archive_dir = (File.dirname f) + '/archive/'
+          Dir.mkdir archive_dir unless Dir.exist? archive_dir
+          File.rename f, archive_dir + (File.basename f)
+        end
+      end
+
+      def read_file(file)
+        d = CSVLib.read file
+        headings = d.first.collect do |v|
+          title = v.gsub(/[^a-z]+/i, '_')
+          title.downcase.intern
+        end
+        d[1..-1].collect { |v| (headings.zip v).to_h }
+      end
+    end
+
+    # Target definition
+    class Target
+      def initialize
+        fail 'Not Implemented'
+      end
+    end
+  end
+end

data/lib/messenger_pigeon/orgmode.rb

@@ -0,0 +1,56 @@
+
+
+module MessengerPigeon
+  # OrgMode source/target module
+  module OrgMode
+    # Source definition
+    class Source
+      def initialize(_options = {})
+        fail 'Not Implemented'
+      end
+    end
+
+    # Target definition
+    class Target
+      def initialize(options = nil)
+        @options = options
+        if File.exist? options[:file]
+          @fd = File.open(options[:file], 'r+')
+          @fd.seek 0
+          @filedata = @fd.read nil
+        else
+          @fd = File.open(options[:file], 'w')
+          @filedata = ''
+        end
+        @known_headings = known_headings
+      end
+
+      def known_headings
+        res = []
+        @filedata.each_line do |l|
+          /^\*+\ (?<heading>.+?)(:[^ ]*:)?$/ =~ l
+          res.push heading if heading
+        end
+        res
+      end
+
+      def update(data)
+        heading = @options[:refile_target] % data
+        data_string = @options[:data_format].call data
+        if @known_headings.include? heading
+          @filedata.sub!(/^(\* #{heading}.*)$/,
+                         "\\1\n#{data_string}")
+        else
+          @filedata += "* #{heading}\n#{data_string}\n"
+          @known_headings.push heading
+        end
+      end
+
+      def write
+        @fd.seek 0
+        @fd.write @filedata
+        @fd.close
+      end
+    end
+  end
+end

data/lib/messenger_pigeon/pigeon.rb

@@ -0,0 +1,59 @@
+
+module MessengerPigeon
+  # Check and apply data matches
+  class Pigeon
+    def initialize(source, target, pigeon_config)
+      @source = source
+      @target = target
+      @filters = pigeon_config[:filters]
+      @transforms = pigeon_config[:transforms]
+      @generators = pigeon_config[:generators]
+    end
+
+    def fly
+      @source.read.each do |d|
+        transforms :pre_filter, d
+        generators d
+        @target.update(transforms :post_filter, d) if filter d
+      end
+      self
+    end
+
+    def finalise
+      @target.write
+      @source.complete
+    end
+
+    private
+
+    def filter(data)
+      return if @filters.nil?
+      @filters.each do |k, v|
+        if v.class == Regexp
+          return false unless data[k] =~ v
+        elsif v.class == String
+          return false unless data[k] == v
+        elsif v.class == Proc
+          return false unless v.call data[k]
+        end
+      end
+      true
+    end
+
+    def transforms(type, data)
+      return data if @transforms.nil? || @transforms[:type].nil?
+      @transforms[type].each do |k, v|
+        data[k] = v.call data[k]
+      end
+      data
+    end
+
+    def generators(data)
+      return data if @generators.nil?
+      @generators.each do |k, v|
+        data[k] = v.call data
+      end
+      data
+    end
+  end
+end

data/lib/messenger_pigeon/redmine.rb

@@ -0,0 +1,78 @@
+require 'active_resource'
+
+module MessengerPigeon
+  # Redmine source/target module
+  module Redmine
+    # Handle JSON returned by Redmine
+    class RedmineFormatter
+      include ActiveResource::Formats::JsonFormat
+
+      def decode(json)
+        remove_root(ActiveSupport::JSON.decode(json))
+      end
+
+      private
+
+      def remove_root(data)
+        # Just return the first value in the hash
+        data.each do |_k, v|
+          return v
+        end
+      end
+    end
+
+    # Source definition
+    class Source
+      def initialize(options)
+        @options = options
+        ar_cl = Class.new ActiveResource::Base
+        o = Object.const_set(options[:resource], ar_cl)
+        o.site = options[:site]
+        o.user = options[:user]
+        o.password = options[:password]
+        o.include_root_in_json = true
+        o.format = RedmineFormatter.new
+        @resource = o
+      end
+
+      def read
+        if @options[:mode] == :specific
+          [@resource.find(@options[:key]).attributes]
+        elsif @options[:mode] == :all
+          res = @resource.find(:all, params: @options[:params])
+          res.map do |r|
+            attrs = r.attributes
+            attrs.each do |k, v|
+              attrs[k] = v.attributes if v.respond_to? :attributes
+            end
+          end
+        end
+      end
+
+      def complete
+      end
+    end
+
+    # Target definition
+    class Target
+      def initialize(options)
+        @options = options
+        ar_cl = Class.new ActiveResource::Base
+        o = Object.const_set(options[:resource], ar_cl)
+        o.site = options[:site]
+        o.user = options[:user]
+        o.password = options[:password]
+        o.include_root_in_json = true
+        @resource = o
+      end
+
+      def update(data)
+        m = @resource.new data
+        $stderr.puts issue.errors.full_messages unless m.save
+      end
+
+      def write
+      end
+    end
+  end
+end

data/lib/messenger_pigeon/version.rb

@@ -0,0 +1,3 @@
+module MessengerPigeon
+  VERSION = '0.1.0'
+end

data/lib/messenger_pigeon.rb

@@ -0,0 +1,38 @@
+require 'optparse'
+
+require 'messenger_pigeon/version'
+require 'messenger_pigeon/pigeon'
+require 'messenger_pigeon/orgmode'
+require 'messenger_pigeon/csv'
+require 'messenger_pigeon/redmine'
+require 'messenger_pigeon/console'
+require 'messenger_pigeon/config'
+require 'messenger_pigeon/cli'
+
+# Messenger Pigeon entry
+module MessengerPigeon
+  module_function
+
+  def flock(opts)
+    Config.load(opts[:config])
+    pigeons = []
+    Config.conf[:pigeons].each do |name, instr|
+      next unless Config.pigeons.empty? || Config.pigeons.include?(name)
+      s = prepare_endpoint :sources, instr[:source]
+      t = prepare_endpoint :targets, instr[:target]
+      pigeon = Pigeon.new s, t, instr
+      pigeons << pigeon.fly
+    end
+    pigeons.each(&:finalise)
+  end
+
+  def prepare_endpoint(endpoint, name)
+    v = Config.conf[endpoint][name]
+    m = v[:type]
+    if endpoint == :sources
+      m::Source.new v[:options]
+    elsif endpoint == :targets
+      m::Target.new v[:options]
+    end
+  end
+end

metadata

@@ -0,0 +1,102 @@
+--- !ruby/object:Gem::Specification
+name: messenger_pigeon
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Chris Mann
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-06-10 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: activeresource
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+description: The goal of MessengerPigeon is to provide a highly-configurable and general
+  mechanism to take data from any number of sources, and copy that data to any number
+  of destinations.  Possibly after some modest filtering and transformations.
+email:
+- chris@bitpattern.com.au
+executables:
+- messenger-pigeon
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE.txt
+- README.md
+- bin/messenger-pigeon
+- lib/messenger_pigeon.rb
+- lib/messenger_pigeon/cli.rb
+- lib/messenger_pigeon/config.rb
+- lib/messenger_pigeon/console.rb
+- lib/messenger_pigeon/csv.rb
+- lib/messenger_pigeon/orgmode.rb
+- lib/messenger_pigeon/pigeon.rb
+- lib/messenger_pigeon/redmine.rb
+- lib/messenger_pigeon/version.rb
+homepage: https://github.com/cshclm/MessengerPigeon
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: A means of getting data from one location to another.
+test_files: []
+has_rdoc: