messenger_pigeon 0.1.0 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +90 -27
- data/bin/messenger-pigeon +2 -1
- data/lib/messenger_pigeon.rb +33 -23
- data/lib/messenger_pigeon/cli.rb +2 -2
- data/lib/messenger_pigeon/config.rb +39 -7
- data/lib/messenger_pigeon/endpoint_manager.rb +33 -0
- data/lib/messenger_pigeon/{console.rb → modules/console.rb} +1 -9
- data/lib/messenger_pigeon/{csv.rb → modules/csv.rb} +3 -3
- data/lib/messenger_pigeon/modules/orgmode.rb +95 -0
- data/lib/messenger_pigeon/{redmine.rb → modules/redmine.rb} +42 -30
- data/lib/messenger_pigeon/modules/sql.rb +40 -0
- data/lib/messenger_pigeon/pigeon.rb +11 -10
- data/lib/messenger_pigeon/version.rb +1 -1
- metadata +43 -12
- data/lib/messenger_pigeon/orgmode.rb +0 -56
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 5540adf1f417f66aeb731db15484686df5b00477
|
4
|
+
data.tar.gz: 985746fdeba649aa6113885917d6f09bd5205b11
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 0f804415db00cd5e77abb607f0d1c421e7e33f3b277cac738b49981d92550dcc53eeb327444b6b41467e57195414d49ee20a451ac1a33ccb9eb22d89dd504ded
|
7
|
+
data.tar.gz: 1d6c22619511fc04f89a475e8989fa34392f076cb171a97dc945a3172ea4aa0eb8181432dc00c380851a95eb55f47156c6d9afb70a08c1373781846898270acd
|
data/README.md
CHANGED
@@ -1,14 +1,35 @@
|
|
1
|
-
# MessengerPigeon
|
1
|
+
# MessengerPigeon [![Travis CI](https://travis-ci.org/cshclm/MessengerPigeon.svg?branch=master)](https://travis-ci.org/cshclm/MessengerPigeon)
|
2
2
|
|
3
3
|
A means of getting data from one location to another.
|
4
4
|
|
5
5
|
The goal of MessengerPigeon is to provide a highly-configurable and adaptable
|
6
|
-
animal to take data from any number of sources, and copy that
|
7
|
-
number of destinations. A pigeon may modify, drop or add to your
|
8
|
-
in transit, but should only do these things when you ask it nicely.
|
6
|
+
animal to take record-based data from any number of sources, and copy that
|
7
|
+
data to any number of destinations. A pigeon may modify, drop or add to your
|
8
|
+
data while in transit, but should only do these things when you ask it nicely.
|
9
9
|
|
10
10
|
MessengerPigeon is under heavy development.
|
11
11
|
|
12
|
+
## Installation
|
13
|
+
```
|
14
|
+
gem install messenger_pigeon
|
15
|
+
```
|
16
|
+
|
17
|
+
## Usage
|
18
|
+
Set up a ~/.messenger-pigeon.rc configuration file (as below) then:
|
19
|
+
```
|
20
|
+
$ messenger-pigeon
|
21
|
+
```
|
22
|
+
|
23
|
+
An alternate configuration file can be specified like so
|
24
|
+
```
|
25
|
+
$ messenger-pigeon -c my-other-configuration-file
|
26
|
+
```
|
27
|
+
|
28
|
+
Finally the names of pigeons can be given to let only those out. If no pigeons are listed, all are let-loose.
|
29
|
+
```
|
30
|
+
$ messenger-pigeon Gemima
|
31
|
+
```
|
32
|
+
|
12
33
|
## Configuration
|
13
34
|
Example Configuration demonstrating some features:
|
14
35
|
|
@@ -19,15 +40,16 @@ Example Configuration demonstrating some features:
|
|
19
40
|
* Data Filtering
|
20
41
|
* Pre and Post-filter data transformations
|
21
42
|
|
22
|
-
~/.
|
43
|
+
~/.messenger-pigeon.rc:
|
23
44
|
```ruby
|
24
45
|
{
|
25
|
-
|
46
|
+
target: {
|
26
47
|
'OrgMode' => {
|
27
|
-
type:
|
48
|
+
type: 'OrgMode',
|
28
49
|
options: {
|
29
50
|
file: 'clocks.org',
|
30
|
-
|
51
|
+
heading_selector: '%{project}',
|
52
|
+
level_separator: ' - ',
|
31
53
|
data_format: proc do |d|
|
32
54
|
clock = ' CLOCK: [%{date} %{start_time}]--[%{date} %{end_time}] => %{duration}'
|
33
55
|
clock % d
|
@@ -35,16 +57,16 @@ Example Configuration demonstrating some features:
|
|
35
57
|
}
|
36
58
|
}
|
37
59
|
},
|
38
|
-
|
60
|
+
source: {
|
39
61
|
'TimeTracker' => {
|
40
|
-
type:
|
62
|
+
type: 'CSV',
|
41
63
|
options: {
|
42
64
|
file_glob: 'test.csv',
|
43
65
|
on_complete: :archive
|
44
66
|
}
|
45
67
|
}
|
46
68
|
},
|
47
|
-
|
69
|
+
pigeon: {
|
48
70
|
'Gemima' => {
|
49
71
|
source: 'TimeTracker',
|
50
72
|
filters: {
|
@@ -59,6 +81,9 @@ Example Configuration demonstrating some features:
|
|
59
81
|
end
|
60
82
|
},
|
61
83
|
post_filter: {}
|
84
|
+
logger: proc do |d|
|
85
|
+
puts format('[Gemima] Logging %shrs on %s', d[:duration], d[:date])
|
86
|
+
end
|
62
87
|
},
|
63
88
|
target: 'OrgMode'
|
64
89
|
}
|
@@ -67,14 +92,20 @@ Example Configuration demonstrating some features:
|
|
67
92
|
```
|
68
93
|
|
69
94
|
## Modules
|
95
|
+
The following modules are available:
|
96
|
+
- CSV (source only)
|
97
|
+
- SQL
|
98
|
+
- Console (target only)
|
99
|
+
- Redmine
|
100
|
+
- OrgMode (target only)
|
70
101
|
|
71
102
|
### CSV
|
72
103
|
#### Source
|
73
104
|
```ruby
|
74
105
|
...
|
75
|
-
|
106
|
+
source: {
|
76
107
|
'TimeTracker' => {
|
77
|
-
type:
|
108
|
+
type: 'CSV',
|
78
109
|
options: {
|
79
110
|
file_glob: 'test.csv',
|
80
111
|
on_complete: :archive
|
@@ -86,15 +117,46 @@ sources: {
|
|
86
117
|
#### Target
|
87
118
|
Not yet implemented
|
88
119
|
|
89
|
-
###
|
120
|
+
### SQL
|
121
|
+
Uses [Sequel](https://github.com/jeremyevans/sequel) to interface to many different types of databases.
|
122
|
+
|
123
|
+
See [the Sequel documentation](http://sequel.jeremyevans.net/rdoc/files/doc/opening_databases_rdoc.html) for listing of supported databases and their connection string formats.
|
90
124
|
#### Source
|
91
|
-
|
125
|
+
```ruby
|
126
|
+
...
|
127
|
+
source: {
|
128
|
+
'Mydatabase' => {
|
129
|
+
type: 'SQL',
|
130
|
+
options: {
|
131
|
+
connection_string: 'sqlite:/',
|
132
|
+
query: "SELECT * FROM artists"
|
133
|
+
}
|
134
|
+
}
|
135
|
+
}
|
136
|
+
...
|
137
|
+
```
|
138
|
+
#### Target
|
139
|
+
```ruby
|
140
|
+
...
|
141
|
+
target: {
|
142
|
+
'Mydatabase' => {
|
143
|
+
type: 'SQL',
|
144
|
+
options: {
|
145
|
+
connection_string: 'sqlite:/',
|
146
|
+
table: 'artists'
|
147
|
+
}
|
148
|
+
}
|
149
|
+
}
|
150
|
+
...
|
151
|
+
```
|
152
|
+
### Console
|
153
|
+
Target-only.
|
92
154
|
#### Target
|
93
155
|
```ruby
|
94
156
|
...
|
95
|
-
|
157
|
+
target: {
|
96
158
|
'Console => {
|
97
|
-
type:
|
159
|
+
type: 'Console',
|
98
160
|
options: {}
|
99
161
|
}
|
100
162
|
}
|
@@ -106,12 +168,13 @@ Not yet implemented
|
|
106
168
|
#### Target
|
107
169
|
```ruby
|
108
170
|
...
|
109
|
-
|
171
|
+
target: {
|
110
172
|
'OrgMode' => {
|
111
|
-
type:
|
173
|
+
type: 'OrgMode',
|
112
174
|
options: {
|
113
175
|
file: 'clocks.org',
|
114
|
-
|
176
|
+
heading_selector: '%{project}',
|
177
|
+
level_separator: ' - ',
|
115
178
|
data_format: proc do |d|
|
116
179
|
clock = ' CLOCK: [%{date} %{start_time}]--[%{date} %{end_time}] => %{duration}'
|
117
180
|
clock % d
|
@@ -135,9 +198,9 @@ E.g., 'time_entries' becomes 'TimeEntry'
|
|
135
198
|
Example for :all to get all issues in Project 2.
|
136
199
|
```ruby
|
137
200
|
...
|
138
|
-
|
201
|
+
source: {
|
139
202
|
'Redmine' => {
|
140
|
-
type:
|
203
|
+
type: 'Redmine',
|
141
204
|
options: {
|
142
205
|
resource: "Issue",
|
143
206
|
site: 'https://url.to.redmine.org',
|
@@ -157,9 +220,9 @@ sources: {
|
|
157
220
|
Example for :specific to get the issue with ID = 30
|
158
221
|
```ruby
|
159
222
|
...
|
160
|
-
|
223
|
+
source: {
|
161
224
|
'Redmine' => {
|
162
|
-
type:
|
225
|
+
type: 'Redmine',
|
163
226
|
options: {
|
164
227
|
resource: "Issue",
|
165
228
|
site: 'https://url.to.redmine.org',
|
@@ -180,9 +243,9 @@ keys required on the particular resource page for more information.
|
|
180
243
|
|
181
244
|
```ruby
|
182
245
|
...
|
183
|
-
|
246
|
+
target: {
|
184
247
|
'Redmine' => {
|
185
|
-
type:
|
248
|
+
type: 'Redmine',
|
186
249
|
options: {
|
187
250
|
resource: "TimeEntry",
|
188
251
|
site: 'https://url.to.redmine.org',
|
@@ -191,7 +254,7 @@ targets: {
|
|
191
254
|
}
|
192
255
|
},
|
193
256
|
},
|
194
|
-
|
257
|
+
pigeon: {
|
195
258
|
'Gemima' => {
|
196
259
|
filters: {
|
197
260
|
issue_id: proc { |a| !a.nil? },
|
data/bin/messenger-pigeon
CHANGED
data/lib/messenger_pigeon.rb
CHANGED
@@ -2,37 +2,47 @@ require 'optparse'
|
|
2
2
|
|
3
3
|
require 'messenger_pigeon/version'
|
4
4
|
require 'messenger_pigeon/pigeon'
|
5
|
-
require 'messenger_pigeon/orgmode'
|
6
|
-
require 'messenger_pigeon/csv'
|
7
|
-
require 'messenger_pigeon/redmine'
|
8
|
-
require 'messenger_pigeon/console'
|
9
5
|
require 'messenger_pigeon/config'
|
6
|
+
require 'messenger_pigeon/endpoint_manager'
|
10
7
|
require 'messenger_pigeon/cli'
|
11
8
|
|
9
|
+
require 'messenger_pigeon/modules/orgmode'
|
10
|
+
require 'messenger_pigeon/modules/console'
|
11
|
+
require 'messenger_pigeon/modules/redmine'
|
12
|
+
require 'messenger_pigeon/modules/csv'
|
13
|
+
|
12
14
|
# Messenger Pigeon entry
|
13
15
|
module MessengerPigeon
|
14
|
-
|
16
|
+
# A flock of Pigeons
|
17
|
+
# Manage configuration and dispatch
|
18
|
+
class Flock
|
19
|
+
def initialize(opts, args)
|
20
|
+
@c = Config.new(opts[:config], args)
|
21
|
+
@endpoints = {
|
22
|
+
source: EndpointManager.new(:source),
|
23
|
+
target: EndpointManager.new(:target)
|
24
|
+
}
|
25
|
+
end
|
15
26
|
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
27
|
+
def release
|
28
|
+
@c.conf[:pigeon].each do |name, instructions|
|
29
|
+
next unless @c.release_pigeon? name
|
30
|
+
release_pigeon prepare_endpoints(instructions), instructions
|
31
|
+
end
|
32
|
+
@endpoints.values.each(&:finalise)
|
33
|
+
end
|
34
|
+
|
35
|
+
private
|
36
|
+
|
37
|
+
def prepare_endpoints(instructions)
|
38
|
+
@endpoints.collect do |k, v|
|
39
|
+
v.prepare instructions[k], @c.conf[k][instructions[k]]
|
40
|
+
end
|
25
41
|
end
|
26
|
-
pigeons.each(&:finalise)
|
27
|
-
end
|
28
42
|
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
if endpoint == :sources
|
33
|
-
m::Source.new v[:options]
|
34
|
-
elsif endpoint == :targets
|
35
|
-
m::Target.new v[:options]
|
43
|
+
def release_pigeon(eps, instructions)
|
44
|
+
pigeon = Pigeon.new(*eps, instructions)
|
45
|
+
pigeon.fly
|
36
46
|
end
|
37
47
|
end
|
38
48
|
end
|
data/lib/messenger_pigeon/cli.rb
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
module MessengerPigeon
|
2
|
+
# Helper for runs from the command line
|
2
3
|
module CLI
|
3
4
|
module_function
|
4
5
|
|
@@ -8,12 +9,11 @@ module MessengerPigeon
|
|
8
9
|
'~/.messenger-pigeon.rc')
|
9
10
|
}
|
10
11
|
OptionParser.new do |opts|
|
11
|
-
opts.banner = "Usage:
|
12
|
+
opts.banner = "Usage: messenger-pigeon [options] [pigeons]"
|
12
13
|
opts.on('-c', '--config file', 'Configuration file') do |f|
|
13
14
|
options[:config] = f
|
14
15
|
end
|
15
16
|
end.parse!
|
16
|
-
Config.pigeons = ARGV
|
17
17
|
options
|
18
18
|
end
|
19
19
|
end
|
@@ -1,14 +1,46 @@
|
|
1
1
|
module MessengerPigeon
|
2
2
|
# MessengerPigeon Configuration
|
3
|
-
|
4
|
-
extend self
|
5
|
-
|
3
|
+
class Config
|
6
4
|
attr_reader :conf
|
7
|
-
attr_accessor :pigeons
|
8
5
|
|
9
|
-
def
|
10
|
-
@conf
|
11
|
-
@
|
6
|
+
def initialize(config, pigeons = [])
|
7
|
+
@conf = {}
|
8
|
+
@pigeons = pigeons
|
9
|
+
update config
|
10
|
+
end
|
11
|
+
|
12
|
+
def release_pigeon?(name)
|
13
|
+
@pigeons.empty? || @pigeons.include?(name)
|
14
|
+
end
|
15
|
+
|
16
|
+
private
|
17
|
+
|
18
|
+
def update(new_conf)
|
19
|
+
temp_conf = @conf.clone
|
20
|
+
case new_conf
|
21
|
+
when String
|
22
|
+
temp_conf.merge!(load_file new_conf)
|
23
|
+
when Hash
|
24
|
+
temp_conf.merge! new_conf
|
25
|
+
else
|
26
|
+
fail "Unsupported configuration datatype: #{new_conf.class}"
|
27
|
+
end
|
28
|
+
apply_config temp_conf
|
29
|
+
end
|
30
|
+
|
31
|
+
# TODO: should do a whole lot more checking. Config pretty expressive and
|
32
|
+
# we want to catch problems ASAP.
|
33
|
+
def apply_config(config)
|
34
|
+
[:pigeon, :target, :source].each do |setting|
|
35
|
+
fail "Configuration invalid: #{setting.to_s}" unless
|
36
|
+
config[setting] && config[setting].class == Hash &&
|
37
|
+
!config[setting].keys.empty?
|
38
|
+
end
|
39
|
+
@conf = config
|
40
|
+
end
|
41
|
+
|
42
|
+
def load_file(config)
|
43
|
+
binding.eval(File.read(File.expand_path(config)))
|
12
44
|
end
|
13
45
|
end
|
14
46
|
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
module MessengerPigeon
|
2
|
+
# Create and manage instances of Sources or Destinations
|
3
|
+
class EndpointManager
|
4
|
+
attr_reader :instances
|
5
|
+
|
6
|
+
def initialize(direction)
|
7
|
+
@instances = {}
|
8
|
+
@direction = direction.to_s.capitalize
|
9
|
+
end
|
10
|
+
|
11
|
+
def prepare(name, endpoint_defn)
|
12
|
+
fail 'Endpoint definition not defined' unless endpoint_defn.class == Hash
|
13
|
+
type = endpoint_defn[:type] || fail('Endpoint type not set')
|
14
|
+
@instances[name] || build_instance(name, type, endpoint_defn[:options])
|
15
|
+
end
|
16
|
+
|
17
|
+
def direction
|
18
|
+
@direction.downcase.to_sym
|
19
|
+
end
|
20
|
+
|
21
|
+
def finalise
|
22
|
+
@instances.values.each(&:complete)
|
23
|
+
@instances = {}
|
24
|
+
end
|
25
|
+
|
26
|
+
private
|
27
|
+
|
28
|
+
def build_instance(name, type, options)
|
29
|
+
o = Kernel.const_get("MessengerPigeon").const_get("#{type}").const_get("#{@direction}")
|
30
|
+
@instances[name] = o.new(options)
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -1,14 +1,6 @@
|
|
1
|
-
|
2
1
|
module MessengerPigeon
|
3
2
|
# Console source/target module
|
4
3
|
module Console
|
5
|
-
# Source definition
|
6
|
-
class Source
|
7
|
-
def initialize
|
8
|
-
fail 'Not Implemented'
|
9
|
-
end
|
10
|
-
end
|
11
|
-
|
12
4
|
# Target definition
|
13
5
|
class Target
|
14
6
|
def initialize(_options)
|
@@ -18,7 +10,7 @@ module MessengerPigeon
|
|
18
10
|
puts data
|
19
11
|
end
|
20
12
|
|
21
|
-
def
|
13
|
+
def complete
|
22
14
|
end
|
23
15
|
end
|
24
16
|
end
|
@@ -7,12 +7,12 @@ module MessengerPigeon
|
|
7
7
|
class Source
|
8
8
|
def initialize(options)
|
9
9
|
# :file_glob
|
10
|
-
@files = Dir.glob(options[:file_glob])
|
10
|
+
@files = Dir.glob(File.expand_path options[:file_glob])
|
11
11
|
@on_complete = options[:on_complete]
|
12
12
|
end
|
13
13
|
|
14
14
|
def read
|
15
|
-
@
|
15
|
+
@files.collect { |f| read_file f }.flatten
|
16
16
|
end
|
17
17
|
|
18
18
|
def complete
|
@@ -28,7 +28,7 @@ module MessengerPigeon
|
|
28
28
|
@files.each do |f|
|
29
29
|
archive_dir = (File.dirname f) + '/archive/'
|
30
30
|
Dir.mkdir archive_dir unless Dir.exist? archive_dir
|
31
|
-
File.rename
|
31
|
+
File.rename(f, archive_dir + (File.basename f)) if File.exist? f
|
32
32
|
end
|
33
33
|
end
|
34
34
|
|
@@ -0,0 +1,95 @@
|
|
1
|
+
|
2
|
+
|
3
|
+
module MessengerPigeon
|
4
|
+
# OrgMode source/target module
|
5
|
+
module OrgMode
|
6
|
+
# Source definition
|
7
|
+
class Source
|
8
|
+
def initialize(_options = {})
|
9
|
+
fail 'Not Implemented'
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
# Target definition
|
14
|
+
class Target
|
15
|
+
def initialize(options = nil)
|
16
|
+
@options = options
|
17
|
+
open_file File.expand_path(options[:file])
|
18
|
+
@known_headings = known_headings
|
19
|
+
end
|
20
|
+
|
21
|
+
def filedata
|
22
|
+
(@filedata.join "\n") + "\n"
|
23
|
+
end
|
24
|
+
|
25
|
+
def known_headings
|
26
|
+
res = []
|
27
|
+
@filedata.each do |l|
|
28
|
+
/^\*+\ (?<heading>.+?)\ +(:[^ ]*:)?$/ =~ l
|
29
|
+
res.push heading if heading
|
30
|
+
end
|
31
|
+
res
|
32
|
+
end
|
33
|
+
|
34
|
+
def find_target(headings, star_count, start_line, end_line)
|
35
|
+
if headings.empty?
|
36
|
+
# Found the target
|
37
|
+
return start_line + 1
|
38
|
+
end
|
39
|
+
target_heading = headings.pop
|
40
|
+
@filedata[start_line..end_line].each do |line|
|
41
|
+
if /^\*{#{star_count}}\ #{target_heading}(\ +(:[^ ]*:)?)?$/ =~ line
|
42
|
+
return find_target(headings, star_count + 1, start_line, find_next_heading(star_count, start_line, end_line))
|
43
|
+
end
|
44
|
+
start_line += 1
|
45
|
+
end
|
46
|
+
# Left-over headings
|
47
|
+
([target_heading] + headings).each do |heading|
|
48
|
+
@filedata.insert end_line, "#{'*' * star_count} #{heading}"
|
49
|
+
star_count += 1
|
50
|
+
end_line += 1
|
51
|
+
end
|
52
|
+
end_line
|
53
|
+
end
|
54
|
+
|
55
|
+
def find_next_heading(star_count, start_line, end_line)
|
56
|
+
@filedata[start_line + 1..end_line].each do |line|
|
57
|
+
if /^\*{#{star_count}}\ (?<heading>.+?)\ +(:[^ ]*:)?$/ =~ line
|
58
|
+
return start_line
|
59
|
+
end
|
60
|
+
start_line += 1
|
61
|
+
end
|
62
|
+
end_line
|
63
|
+
end
|
64
|
+
|
65
|
+
def update(data)
|
66
|
+
selector = @options[:heading_selector] % data
|
67
|
+
headings = selector.split(@options[:level_separator])
|
68
|
+
star_count = 1
|
69
|
+
target = find_target headings.reverse, star_count, 0, @filedata.length
|
70
|
+
data_string = @options[:data_format].call data
|
71
|
+
@filedata.insert target, data_string
|
72
|
+
end
|
73
|
+
|
74
|
+
def complete
|
75
|
+
@fd.seek 0
|
76
|
+
@fd.write filedata
|
77
|
+
@fd.close
|
78
|
+
end
|
79
|
+
|
80
|
+
private
|
81
|
+
|
82
|
+
def open_file file
|
83
|
+
if File.exist? file
|
84
|
+
@fd = File.open(file, 'r+')
|
85
|
+
@fd.seek 0
|
86
|
+
@filedata = @fd.read nil
|
87
|
+
else
|
88
|
+
@fd = File.open(file, 'w')
|
89
|
+
@filedata = ''
|
90
|
+
end
|
91
|
+
@filedata = @filedata.split(/[\n\r]/)
|
92
|
+
end
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
@@ -3,8 +3,32 @@ require 'active_resource'
|
|
3
3
|
module MessengerPigeon
|
4
4
|
# Redmine source/target module
|
5
5
|
module Redmine
|
6
|
+
# Module mixin for redmine activeresource init
|
7
|
+
module Init
|
8
|
+
@@known_instances = {}
|
9
|
+
|
10
|
+
def initialize(options)
|
11
|
+
@options = options
|
12
|
+
o = get_resource options[:resource]
|
13
|
+
o.site = options[:site]
|
14
|
+
o.user = options[:user]
|
15
|
+
o.password = options[:password]
|
16
|
+
o.include_root_in_json = true
|
17
|
+
o.format = JsonFormatter.new
|
18
|
+
@resource = o
|
19
|
+
end
|
20
|
+
|
21
|
+
private
|
22
|
+
|
23
|
+
def get_resource(name)
|
24
|
+
return @@known_instances[name] unless @@known_instances[name].nil?
|
25
|
+
ar_cl = Class.new ActiveResource::Base
|
26
|
+
@@known_instances[name] = Object.const_set(name, ar_cl)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
6
30
|
# Handle JSON returned by Redmine
|
7
|
-
class
|
31
|
+
class JsonFormatter
|
8
32
|
include ActiveResource::Formats::JsonFormat
|
9
33
|
|
10
34
|
def decode(json)
|
@@ -23,55 +47,43 @@ module MessengerPigeon
|
|
23
47
|
|
24
48
|
# Source definition
|
25
49
|
class Source
|
26
|
-
|
27
|
-
@options = options
|
28
|
-
ar_cl = Class.new ActiveResource::Base
|
29
|
-
o = Object.const_set(options[:resource], ar_cl)
|
30
|
-
o.site = options[:site]
|
31
|
-
o.user = options[:user]
|
32
|
-
o.password = options[:password]
|
33
|
-
o.include_root_in_json = true
|
34
|
-
o.format = RedmineFormatter.new
|
35
|
-
@resource = o
|
36
|
-
end
|
50
|
+
include Init
|
37
51
|
|
38
52
|
def read
|
39
53
|
if @options[:mode] == :specific
|
40
54
|
[@resource.find(@options[:key]).attributes]
|
41
55
|
elsif @options[:mode] == :all
|
42
|
-
|
43
|
-
res.map do |r|
|
44
|
-
attrs = r.attributes
|
45
|
-
attrs.each do |k, v|
|
46
|
-
attrs[k] = v.attributes if v.respond_to? :attributes
|
47
|
-
end
|
48
|
-
end
|
56
|
+
read_all
|
49
57
|
end
|
50
58
|
end
|
51
59
|
|
52
60
|
def complete
|
53
61
|
end
|
62
|
+
|
63
|
+
private
|
64
|
+
|
65
|
+
def read_all
|
66
|
+
res = @resource.find(:all, params: @options[:params])
|
67
|
+
res.map do |r|
|
68
|
+
attrs = r.attributes
|
69
|
+
attrs.each do |k, v|
|
70
|
+
# Convert 1 level of nested attributes
|
71
|
+
attrs[k] = v.attributes if v.respond_to? :attributes
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
54
75
|
end
|
55
76
|
|
56
77
|
# Target definition
|
57
78
|
class Target
|
58
|
-
|
59
|
-
@options = options
|
60
|
-
ar_cl = Class.new ActiveResource::Base
|
61
|
-
o = Object.const_set(options[:resource], ar_cl)
|
62
|
-
o.site = options[:site]
|
63
|
-
o.user = options[:user]
|
64
|
-
o.password = options[:password]
|
65
|
-
o.include_root_in_json = true
|
66
|
-
@resource = o
|
67
|
-
end
|
79
|
+
include Init
|
68
80
|
|
69
81
|
def update(data)
|
70
82
|
m = @resource.new data
|
71
83
|
$stderr.puts issue.errors.full_messages unless m.save
|
72
84
|
end
|
73
85
|
|
74
|
-
def
|
86
|
+
def complete
|
75
87
|
end
|
76
88
|
end
|
77
89
|
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
require 'sequel'
|
2
|
+
|
3
|
+
module MessengerPigeon
|
4
|
+
# Redmine source/target module
|
5
|
+
module SQL
|
6
|
+
# Source definition
|
7
|
+
module Init
|
8
|
+
attr_accessor :conn
|
9
|
+
|
10
|
+
def initialize(options)
|
11
|
+
@options = options
|
12
|
+
cs = options[:connection_string]
|
13
|
+
@conn = Sequel.connect(cs)
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
class Source
|
18
|
+
include Init
|
19
|
+
|
20
|
+
def read
|
21
|
+
@conn.fetch @options[:query]
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
# Module mixin for redmine activeresource init
|
26
|
+
class Target
|
27
|
+
include Init
|
28
|
+
|
29
|
+
def update(data)
|
30
|
+
tbl = @conn[@options[:table].intern]
|
31
|
+
data.each do |row|
|
32
|
+
tbl.insert row
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def complete
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
@@ -8,32 +8,33 @@ module MessengerPigeon
|
|
8
8
|
@filters = pigeon_config[:filters]
|
9
9
|
@transforms = pigeon_config[:transforms]
|
10
10
|
@generators = pigeon_config[:generators]
|
11
|
+
@logger = pigeon_config[:logger]
|
11
12
|
end
|
12
13
|
|
13
14
|
def fly
|
14
15
|
@source.read.each do |d|
|
15
16
|
transforms :pre_filter, d
|
16
17
|
generators d
|
17
|
-
|
18
|
+
if filter d
|
19
|
+
d = transforms :post_filter, d
|
20
|
+
@target.update d
|
21
|
+
@logger.call d if @logger
|
22
|
+
end
|
18
23
|
end
|
19
24
|
self
|
20
25
|
end
|
21
26
|
|
22
|
-
def finalise
|
23
|
-
@target.write
|
24
|
-
@source.complete
|
25
|
-
end
|
26
|
-
|
27
27
|
private
|
28
28
|
|
29
29
|
def filter(data)
|
30
30
|
return if @filters.nil?
|
31
31
|
@filters.each do |k, v|
|
32
|
-
|
32
|
+
case v
|
33
|
+
when Regexp
|
33
34
|
return false unless data[k] =~ v
|
34
|
-
|
35
|
+
when String
|
35
36
|
return false unless data[k] == v
|
36
|
-
|
37
|
+
when Proc
|
37
38
|
return false unless v.call data[k]
|
38
39
|
end
|
39
40
|
end
|
@@ -41,7 +42,7 @@ module MessengerPigeon
|
|
41
42
|
end
|
42
43
|
|
43
44
|
def transforms(type, data)
|
44
|
-
return data if @transforms.nil? || @transforms[
|
45
|
+
return data if @transforms.nil? || @transforms[type].nil?
|
45
46
|
@transforms[type].each do |k, v|
|
46
47
|
data[k] = v.call data[k]
|
47
48
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: messenger_pigeon
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Chris Mann
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-09-27 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -42,19 +42,48 @@ dependencies:
|
|
42
42
|
name: activeresource
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
|
-
- -
|
45
|
+
- - '='
|
46
46
|
- !ruby/object:Gem::Version
|
47
47
|
version: '4.0'
|
48
48
|
type: :development
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
|
-
- -
|
52
|
+
- - '='
|
53
53
|
- !ruby/object:Gem::Version
|
54
54
|
version: '4.0'
|
55
|
-
|
56
|
-
|
57
|
-
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: sequel
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '4.26'
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '4.26'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: sqlite3
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - "~>"
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '1.3'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - "~>"
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '1.3'
|
83
|
+
description: The goal of MessengerPigeon is to provide a highly-configurable and adaptable
|
84
|
+
animal to take record-based data from any number of sources, and copy that data
|
85
|
+
to any number of destinations. A pigeon may modify, drop or add to your data while
|
86
|
+
in transit, but should only do these things when you ask it nicely.
|
58
87
|
email:
|
59
88
|
- chris@bitpattern.com.au
|
60
89
|
executables:
|
@@ -68,11 +97,13 @@ files:
|
|
68
97
|
- lib/messenger_pigeon.rb
|
69
98
|
- lib/messenger_pigeon/cli.rb
|
70
99
|
- lib/messenger_pigeon/config.rb
|
71
|
-
- lib/messenger_pigeon/
|
72
|
-
- lib/messenger_pigeon/
|
73
|
-
- lib/messenger_pigeon/
|
100
|
+
- lib/messenger_pigeon/endpoint_manager.rb
|
101
|
+
- lib/messenger_pigeon/modules/console.rb
|
102
|
+
- lib/messenger_pigeon/modules/csv.rb
|
103
|
+
- lib/messenger_pigeon/modules/orgmode.rb
|
104
|
+
- lib/messenger_pigeon/modules/redmine.rb
|
105
|
+
- lib/messenger_pigeon/modules/sql.rb
|
74
106
|
- lib/messenger_pigeon/pigeon.rb
|
75
|
-
- lib/messenger_pigeon/redmine.rb
|
76
107
|
- lib/messenger_pigeon/version.rb
|
77
108
|
homepage: https://github.com/cshclm/MessengerPigeon
|
78
109
|
licenses:
|
@@ -86,7 +117,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
86
117
|
requirements:
|
87
118
|
- - ">="
|
88
119
|
- !ruby/object:Gem::Version
|
89
|
-
version:
|
120
|
+
version: 1.9.3
|
90
121
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
91
122
|
requirements:
|
92
123
|
- - ">="
|
@@ -1,56 +0,0 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
module MessengerPigeon
|
4
|
-
# OrgMode source/target module
|
5
|
-
module OrgMode
|
6
|
-
# Source definition
|
7
|
-
class Source
|
8
|
-
def initialize(_options = {})
|
9
|
-
fail 'Not Implemented'
|
10
|
-
end
|
11
|
-
end
|
12
|
-
|
13
|
-
# Target definition
|
14
|
-
class Target
|
15
|
-
def initialize(options = nil)
|
16
|
-
@options = options
|
17
|
-
if File.exist? options[:file]
|
18
|
-
@fd = File.open(options[:file], 'r+')
|
19
|
-
@fd.seek 0
|
20
|
-
@filedata = @fd.read nil
|
21
|
-
else
|
22
|
-
@fd = File.open(options[:file], 'w')
|
23
|
-
@filedata = ''
|
24
|
-
end
|
25
|
-
@known_headings = known_headings
|
26
|
-
end
|
27
|
-
|
28
|
-
def known_headings
|
29
|
-
res = []
|
30
|
-
@filedata.each_line do |l|
|
31
|
-
/^\*+\ (?<heading>.+?)(:[^ ]*:)?$/ =~ l
|
32
|
-
res.push heading if heading
|
33
|
-
end
|
34
|
-
res
|
35
|
-
end
|
36
|
-
|
37
|
-
def update(data)
|
38
|
-
heading = @options[:refile_target] % data
|
39
|
-
data_string = @options[:data_format].call data
|
40
|
-
if @known_headings.include? heading
|
41
|
-
@filedata.sub!(/^(\* #{heading}.*)$/,
|
42
|
-
"\\1\n#{data_string}")
|
43
|
-
else
|
44
|
-
@filedata += "* #{heading}\n#{data_string}\n"
|
45
|
-
@known_headings.push heading
|
46
|
-
end
|
47
|
-
end
|
48
|
-
|
49
|
-
def write
|
50
|
-
@fd.seek 0
|
51
|
-
@fd.write @filedata
|
52
|
-
@fd.close
|
53
|
-
end
|
54
|
-
end
|
55
|
-
end
|
56
|
-
end
|