messenger_pigeon 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +90 -27
- data/bin/messenger-pigeon +2 -1
- data/lib/messenger_pigeon.rb +33 -23
- data/lib/messenger_pigeon/cli.rb +2 -2
- data/lib/messenger_pigeon/config.rb +39 -7
- data/lib/messenger_pigeon/endpoint_manager.rb +33 -0
- data/lib/messenger_pigeon/{console.rb → modules/console.rb} +1 -9
- data/lib/messenger_pigeon/{csv.rb → modules/csv.rb} +3 -3
- data/lib/messenger_pigeon/modules/orgmode.rb +95 -0
- data/lib/messenger_pigeon/{redmine.rb → modules/redmine.rb} +42 -30
- data/lib/messenger_pigeon/modules/sql.rb +40 -0
- data/lib/messenger_pigeon/pigeon.rb +11 -10
- data/lib/messenger_pigeon/version.rb +1 -1
- metadata +43 -12
- data/lib/messenger_pigeon/orgmode.rb +0 -56
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 5540adf1f417f66aeb731db15484686df5b00477
|
4
|
+
data.tar.gz: 985746fdeba649aa6113885917d6f09bd5205b11
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 0f804415db00cd5e77abb607f0d1c421e7e33f3b277cac738b49981d92550dcc53eeb327444b6b41467e57195414d49ee20a451ac1a33ccb9eb22d89dd504ded
|
7
|
+
data.tar.gz: 1d6c22619511fc04f89a475e8989fa34392f076cb171a97dc945a3172ea4aa0eb8181432dc00c380851a95eb55f47156c6d9afb70a08c1373781846898270acd
|
data/README.md
CHANGED
@@ -1,14 +1,35 @@
|
|
1
|
-
# MessengerPigeon
|
1
|
+
# MessengerPigeon [](https://travis-ci.org/cshclm/MessengerPigeon)
|
2
2
|
|
3
3
|
A means of getting data from one location to another.
|
4
4
|
|
5
5
|
The goal of MessengerPigeon is to provide a highly-configurable and adaptable
|
6
|
-
animal to take data from any number of sources, and copy that
|
7
|
-
number of destinations. A pigeon may modify, drop or add to your
|
8
|
-
in transit, but should only do these things when you ask it nicely.
|
6
|
+
animal to take record-based data from any number of sources, and copy that
|
7
|
+
data to any number of destinations. A pigeon may modify, drop or add to your
|
8
|
+
data while in transit, but should only do these things when you ask it nicely.
|
9
9
|
|
10
10
|
MessengerPigeon is under heavy development.
|
11
11
|
|
12
|
+
## Installation
|
13
|
+
```
|
14
|
+
gem install messenger_pigeon
|
15
|
+
```
|
16
|
+
|
17
|
+
## Usage
|
18
|
+
Set up a ~/.messenger-pigeon.rc configuration file (as below) then:
|
19
|
+
```
|
20
|
+
$ messenger-pigeon
|
21
|
+
```
|
22
|
+
|
23
|
+
An alternate configuration file can be specified like so
|
24
|
+
```
|
25
|
+
$ messenger-pigeon -c my-other-configuration-file
|
26
|
+
```
|
27
|
+
|
28
|
+
Finally the names of pigeons can be given to let only those out. If no pigeons are listed, all are let-loose.
|
29
|
+
```
|
30
|
+
$ messenger-pigeon Gemima
|
31
|
+
```
|
32
|
+
|
12
33
|
## Configuration
|
13
34
|
Example Configuration demonstrating some features:
|
14
35
|
|
@@ -19,15 +40,16 @@ Example Configuration demonstrating some features:
|
|
19
40
|
* Data Filtering
|
20
41
|
* Pre and Post-filter data transformations
|
21
42
|
|
22
|
-
~/.
|
43
|
+
~/.messenger-pigeon.rc:
|
23
44
|
```ruby
|
24
45
|
{
|
25
|
-
|
46
|
+
target: {
|
26
47
|
'OrgMode' => {
|
27
|
-
type:
|
48
|
+
type: 'OrgMode',
|
28
49
|
options: {
|
29
50
|
file: 'clocks.org',
|
30
|
-
|
51
|
+
heading_selector: '%{project}',
|
52
|
+
level_separator: ' - ',
|
31
53
|
data_format: proc do |d|
|
32
54
|
clock = ' CLOCK: [%{date} %{start_time}]--[%{date} %{end_time}] => %{duration}'
|
33
55
|
clock % d
|
@@ -35,16 +57,16 @@ Example Configuration demonstrating some features:
|
|
35
57
|
}
|
36
58
|
}
|
37
59
|
},
|
38
|
-
|
60
|
+
source: {
|
39
61
|
'TimeTracker' => {
|
40
|
-
type:
|
62
|
+
type: 'CSV',
|
41
63
|
options: {
|
42
64
|
file_glob: 'test.csv',
|
43
65
|
on_complete: :archive
|
44
66
|
}
|
45
67
|
}
|
46
68
|
},
|
47
|
-
|
69
|
+
pigeon: {
|
48
70
|
'Gemima' => {
|
49
71
|
source: 'TimeTracker',
|
50
72
|
filters: {
|
@@ -59,6 +81,9 @@ Example Configuration demonstrating some features:
|
|
59
81
|
end
|
60
82
|
},
|
61
83
|
post_filter: {}
|
84
|
+
logger: proc do |d|
|
85
|
+
puts format('[Gemima] Logging %shrs on %s', d[:duration], d[:date])
|
86
|
+
end
|
62
87
|
},
|
63
88
|
target: 'OrgMode'
|
64
89
|
}
|
@@ -67,14 +92,20 @@ Example Configuration demonstrating some features:
|
|
67
92
|
```
|
68
93
|
|
69
94
|
## Modules
|
95
|
+
The following modules are available:
|
96
|
+
- CSV (source only)
|
97
|
+
- SQL
|
98
|
+
- Console (target only)
|
99
|
+
- Redmine
|
100
|
+
- OrgMode (target only)
|
70
101
|
|
71
102
|
### CSV
|
72
103
|
#### Source
|
73
104
|
```ruby
|
74
105
|
...
|
75
|
-
|
106
|
+
source: {
|
76
107
|
'TimeTracker' => {
|
77
|
-
type:
|
108
|
+
type: 'CSV',
|
78
109
|
options: {
|
79
110
|
file_glob: 'test.csv',
|
80
111
|
on_complete: :archive
|
@@ -86,15 +117,46 @@ sources: {
|
|
86
117
|
#### Target
|
87
118
|
Not yet implemented
|
88
119
|
|
89
|
-
###
|
120
|
+
### SQL
|
121
|
+
Uses [Sequel](https://github.com/jeremyevans/sequel) to interface to many different types of databases.
|
122
|
+
|
123
|
+
See [the Sequel documentation](http://sequel.jeremyevans.net/rdoc/files/doc/opening_databases_rdoc.html) for listing of supported databases and their connection string formats.
|
90
124
|
#### Source
|
91
|
-
|
125
|
+
```ruby
|
126
|
+
...
|
127
|
+
source: {
|
128
|
+
'Mydatabase' => {
|
129
|
+
type: 'SQL',
|
130
|
+
options: {
|
131
|
+
connection_string: 'sqlite:/',
|
132
|
+
query: "SELECT * FROM artists"
|
133
|
+
}
|
134
|
+
}
|
135
|
+
}
|
136
|
+
...
|
137
|
+
```
|
138
|
+
#### Target
|
139
|
+
```ruby
|
140
|
+
...
|
141
|
+
target: {
|
142
|
+
'Mydatabase' => {
|
143
|
+
type: 'SQL',
|
144
|
+
options: {
|
145
|
+
connection_string: 'sqlite:/',
|
146
|
+
table: 'artists'
|
147
|
+
}
|
148
|
+
}
|
149
|
+
}
|
150
|
+
...
|
151
|
+
```
|
152
|
+
### Console
|
153
|
+
Target-only.
|
92
154
|
#### Target
|
93
155
|
```ruby
|
94
156
|
...
|
95
|
-
|
157
|
+
target: {
|
96
158
|
'Console => {
|
97
|
-
type:
|
159
|
+
type: 'Console',
|
98
160
|
options: {}
|
99
161
|
}
|
100
162
|
}
|
@@ -106,12 +168,13 @@ Not yet implemented
|
|
106
168
|
#### Target
|
107
169
|
```ruby
|
108
170
|
...
|
109
|
-
|
171
|
+
target: {
|
110
172
|
'OrgMode' => {
|
111
|
-
type:
|
173
|
+
type: 'OrgMode',
|
112
174
|
options: {
|
113
175
|
file: 'clocks.org',
|
114
|
-
|
176
|
+
heading_selector: '%{project}',
|
177
|
+
level_separator: ' - ',
|
115
178
|
data_format: proc do |d|
|
116
179
|
clock = ' CLOCK: [%{date} %{start_time}]--[%{date} %{end_time}] => %{duration}'
|
117
180
|
clock % d
|
@@ -135,9 +198,9 @@ E.g., 'time_entries' becomes 'TimeEntry'
|
|
135
198
|
Example for :all to get all issues in Project 2.
|
136
199
|
```ruby
|
137
200
|
...
|
138
|
-
|
201
|
+
source: {
|
139
202
|
'Redmine' => {
|
140
|
-
type:
|
203
|
+
type: 'Redmine',
|
141
204
|
options: {
|
142
205
|
resource: "Issue",
|
143
206
|
site: 'https://url.to.redmine.org',
|
@@ -157,9 +220,9 @@ sources: {
|
|
157
220
|
Example for :specific to get the issue with ID = 30
|
158
221
|
```ruby
|
159
222
|
...
|
160
|
-
|
223
|
+
source: {
|
161
224
|
'Redmine' => {
|
162
|
-
type:
|
225
|
+
type: 'Redmine',
|
163
226
|
options: {
|
164
227
|
resource: "Issue",
|
165
228
|
site: 'https://url.to.redmine.org',
|
@@ -180,9 +243,9 @@ keys required on the particular resource page for more information.
|
|
180
243
|
|
181
244
|
```ruby
|
182
245
|
...
|
183
|
-
|
246
|
+
target: {
|
184
247
|
'Redmine' => {
|
185
|
-
type:
|
248
|
+
type: 'Redmine',
|
186
249
|
options: {
|
187
250
|
resource: "TimeEntry",
|
188
251
|
site: 'https://url.to.redmine.org',
|
@@ -191,7 +254,7 @@ targets: {
|
|
191
254
|
}
|
192
255
|
},
|
193
256
|
},
|
194
|
-
|
257
|
+
pigeon: {
|
195
258
|
'Gemima' => {
|
196
259
|
filters: {
|
197
260
|
issue_id: proc { |a| !a.nil? },
|
data/bin/messenger-pigeon
CHANGED
data/lib/messenger_pigeon.rb
CHANGED
@@ -2,37 +2,47 @@ require 'optparse'
|
|
2
2
|
|
3
3
|
require 'messenger_pigeon/version'
|
4
4
|
require 'messenger_pigeon/pigeon'
|
5
|
-
require 'messenger_pigeon/orgmode'
|
6
|
-
require 'messenger_pigeon/csv'
|
7
|
-
require 'messenger_pigeon/redmine'
|
8
|
-
require 'messenger_pigeon/console'
|
9
5
|
require 'messenger_pigeon/config'
|
6
|
+
require 'messenger_pigeon/endpoint_manager'
|
10
7
|
require 'messenger_pigeon/cli'
|
11
8
|
|
9
|
+
require 'messenger_pigeon/modules/orgmode'
|
10
|
+
require 'messenger_pigeon/modules/console'
|
11
|
+
require 'messenger_pigeon/modules/redmine'
|
12
|
+
require 'messenger_pigeon/modules/csv'
|
13
|
+
|
12
14
|
# Messenger Pigeon entry
|
13
15
|
module MessengerPigeon
|
14
|
-
|
16
|
+
# A flock of Pigeons
|
17
|
+
# Manage configuration and dispatch
|
18
|
+
class Flock
|
19
|
+
def initialize(opts, args)
|
20
|
+
@c = Config.new(opts[:config], args)
|
21
|
+
@endpoints = {
|
22
|
+
source: EndpointManager.new(:source),
|
23
|
+
target: EndpointManager.new(:target)
|
24
|
+
}
|
25
|
+
end
|
15
26
|
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
27
|
+
def release
|
28
|
+
@c.conf[:pigeon].each do |name, instructions|
|
29
|
+
next unless @c.release_pigeon? name
|
30
|
+
release_pigeon prepare_endpoints(instructions), instructions
|
31
|
+
end
|
32
|
+
@endpoints.values.each(&:finalise)
|
33
|
+
end
|
34
|
+
|
35
|
+
private
|
36
|
+
|
37
|
+
def prepare_endpoints(instructions)
|
38
|
+
@endpoints.collect do |k, v|
|
39
|
+
v.prepare instructions[k], @c.conf[k][instructions[k]]
|
40
|
+
end
|
25
41
|
end
|
26
|
-
pigeons.each(&:finalise)
|
27
|
-
end
|
28
42
|
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
if endpoint == :sources
|
33
|
-
m::Source.new v[:options]
|
34
|
-
elsif endpoint == :targets
|
35
|
-
m::Target.new v[:options]
|
43
|
+
def release_pigeon(eps, instructions)
|
44
|
+
pigeon = Pigeon.new(*eps, instructions)
|
45
|
+
pigeon.fly
|
36
46
|
end
|
37
47
|
end
|
38
48
|
end
|
data/lib/messenger_pigeon/cli.rb
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
module MessengerPigeon
|
2
|
+
# Helper for runs from the command line
|
2
3
|
module CLI
|
3
4
|
module_function
|
4
5
|
|
@@ -8,12 +9,11 @@ module MessengerPigeon
|
|
8
9
|
'~/.messenger-pigeon.rc')
|
9
10
|
}
|
10
11
|
OptionParser.new do |opts|
|
11
|
-
opts.banner = "Usage:
|
12
|
+
opts.banner = "Usage: messenger-pigeon [options] [pigeons]"
|
12
13
|
opts.on('-c', '--config file', 'Configuration file') do |f|
|
13
14
|
options[:config] = f
|
14
15
|
end
|
15
16
|
end.parse!
|
16
|
-
Config.pigeons = ARGV
|
17
17
|
options
|
18
18
|
end
|
19
19
|
end
|
@@ -1,14 +1,46 @@
|
|
1
1
|
module MessengerPigeon
|
2
2
|
# MessengerPigeon Configuration
|
3
|
-
|
4
|
-
extend self
|
5
|
-
|
3
|
+
class Config
|
6
4
|
attr_reader :conf
|
7
|
-
attr_accessor :pigeons
|
8
5
|
|
9
|
-
def
|
10
|
-
@conf
|
11
|
-
@
|
6
|
+
def initialize(config, pigeons = [])
|
7
|
+
@conf = {}
|
8
|
+
@pigeons = pigeons
|
9
|
+
update config
|
10
|
+
end
|
11
|
+
|
12
|
+
def release_pigeon?(name)
|
13
|
+
@pigeons.empty? || @pigeons.include?(name)
|
14
|
+
end
|
15
|
+
|
16
|
+
private
|
17
|
+
|
18
|
+
def update(new_conf)
|
19
|
+
temp_conf = @conf.clone
|
20
|
+
case new_conf
|
21
|
+
when String
|
22
|
+
temp_conf.merge!(load_file new_conf)
|
23
|
+
when Hash
|
24
|
+
temp_conf.merge! new_conf
|
25
|
+
else
|
26
|
+
fail "Unsupported configuration datatype: #{new_conf.class}"
|
27
|
+
end
|
28
|
+
apply_config temp_conf
|
29
|
+
end
|
30
|
+
|
31
|
+
# TODO: should do a whole lot more checking. Config pretty expressive and
|
32
|
+
# we want to catch problems ASAP.
|
33
|
+
def apply_config(config)
|
34
|
+
[:pigeon, :target, :source].each do |setting|
|
35
|
+
fail "Configuration invalid: #{setting.to_s}" unless
|
36
|
+
config[setting] && config[setting].class == Hash &&
|
37
|
+
!config[setting].keys.empty?
|
38
|
+
end
|
39
|
+
@conf = config
|
40
|
+
end
|
41
|
+
|
42
|
+
def load_file(config)
|
43
|
+
binding.eval(File.read(File.expand_path(config)))
|
12
44
|
end
|
13
45
|
end
|
14
46
|
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
module MessengerPigeon
|
2
|
+
# Create and manage instances of Sources or Destinations
|
3
|
+
class EndpointManager
|
4
|
+
attr_reader :instances
|
5
|
+
|
6
|
+
def initialize(direction)
|
7
|
+
@instances = {}
|
8
|
+
@direction = direction.to_s.capitalize
|
9
|
+
end
|
10
|
+
|
11
|
+
def prepare(name, endpoint_defn)
|
12
|
+
fail 'Endpoint definition not defined' unless endpoint_defn.class == Hash
|
13
|
+
type = endpoint_defn[:type] || fail('Endpoint type not set')
|
14
|
+
@instances[name] || build_instance(name, type, endpoint_defn[:options])
|
15
|
+
end
|
16
|
+
|
17
|
+
def direction
|
18
|
+
@direction.downcase.to_sym
|
19
|
+
end
|
20
|
+
|
21
|
+
def finalise
|
22
|
+
@instances.values.each(&:complete)
|
23
|
+
@instances = {}
|
24
|
+
end
|
25
|
+
|
26
|
+
private
|
27
|
+
|
28
|
+
def build_instance(name, type, options)
|
29
|
+
o = Kernel.const_get("MessengerPigeon").const_get("#{type}").const_get("#{@direction}")
|
30
|
+
@instances[name] = o.new(options)
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -1,14 +1,6 @@
|
|
1
|
-
|
2
1
|
module MessengerPigeon
|
3
2
|
# Console source/target module
|
4
3
|
module Console
|
5
|
-
# Source definition
|
6
|
-
class Source
|
7
|
-
def initialize
|
8
|
-
fail 'Not Implemented'
|
9
|
-
end
|
10
|
-
end
|
11
|
-
|
12
4
|
# Target definition
|
13
5
|
class Target
|
14
6
|
def initialize(_options)
|
@@ -18,7 +10,7 @@ module MessengerPigeon
|
|
18
10
|
puts data
|
19
11
|
end
|
20
12
|
|
21
|
-
def
|
13
|
+
def complete
|
22
14
|
end
|
23
15
|
end
|
24
16
|
end
|
@@ -7,12 +7,12 @@ module MessengerPigeon
|
|
7
7
|
class Source
|
8
8
|
def initialize(options)
|
9
9
|
# :file_glob
|
10
|
-
@files = Dir.glob(options[:file_glob])
|
10
|
+
@files = Dir.glob(File.expand_path options[:file_glob])
|
11
11
|
@on_complete = options[:on_complete]
|
12
12
|
end
|
13
13
|
|
14
14
|
def read
|
15
|
-
@
|
15
|
+
@files.collect { |f| read_file f }.flatten
|
16
16
|
end
|
17
17
|
|
18
18
|
def complete
|
@@ -28,7 +28,7 @@ module MessengerPigeon
|
|
28
28
|
@files.each do |f|
|
29
29
|
archive_dir = (File.dirname f) + '/archive/'
|
30
30
|
Dir.mkdir archive_dir unless Dir.exist? archive_dir
|
31
|
-
File.rename
|
31
|
+
File.rename(f, archive_dir + (File.basename f)) if File.exist? f
|
32
32
|
end
|
33
33
|
end
|
34
34
|
|
@@ -0,0 +1,95 @@
|
|
1
|
+
|
2
|
+
|
3
|
+
module MessengerPigeon
|
4
|
+
# OrgMode source/target module
|
5
|
+
module OrgMode
|
6
|
+
# Source definition
|
7
|
+
class Source
|
8
|
+
def initialize(_options = {})
|
9
|
+
fail 'Not Implemented'
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
# Target definition
|
14
|
+
class Target
|
15
|
+
def initialize(options = nil)
|
16
|
+
@options = options
|
17
|
+
open_file File.expand_path(options[:file])
|
18
|
+
@known_headings = known_headings
|
19
|
+
end
|
20
|
+
|
21
|
+
def filedata
|
22
|
+
(@filedata.join "\n") + "\n"
|
23
|
+
end
|
24
|
+
|
25
|
+
def known_headings
|
26
|
+
res = []
|
27
|
+
@filedata.each do |l|
|
28
|
+
/^\*+\ (?<heading>.+?)\ +(:[^ ]*:)?$/ =~ l
|
29
|
+
res.push heading if heading
|
30
|
+
end
|
31
|
+
res
|
32
|
+
end
|
33
|
+
|
34
|
+
def find_target(headings, star_count, start_line, end_line)
|
35
|
+
if headings.empty?
|
36
|
+
# Found the target
|
37
|
+
return start_line + 1
|
38
|
+
end
|
39
|
+
target_heading = headings.pop
|
40
|
+
@filedata[start_line..end_line].each do |line|
|
41
|
+
if /^\*{#{star_count}}\ #{target_heading}(\ +(:[^ ]*:)?)?$/ =~ line
|
42
|
+
return find_target(headings, star_count + 1, start_line, find_next_heading(star_count, start_line, end_line))
|
43
|
+
end
|
44
|
+
start_line += 1
|
45
|
+
end
|
46
|
+
# Left-over headings
|
47
|
+
([target_heading] + headings).each do |heading|
|
48
|
+
@filedata.insert end_line, "#{'*' * star_count} #{heading}"
|
49
|
+
star_count += 1
|
50
|
+
end_line += 1
|
51
|
+
end
|
52
|
+
end_line
|
53
|
+
end
|
54
|
+
|
55
|
+
def find_next_heading(star_count, start_line, end_line)
|
56
|
+
@filedata[start_line + 1..end_line].each do |line|
|
57
|
+
if /^\*{#{star_count}}\ (?<heading>.+?)\ +(:[^ ]*:)?$/ =~ line
|
58
|
+
return start_line
|
59
|
+
end
|
60
|
+
start_line += 1
|
61
|
+
end
|
62
|
+
end_line
|
63
|
+
end
|
64
|
+
|
65
|
+
def update(data)
|
66
|
+
selector = @options[:heading_selector] % data
|
67
|
+
headings = selector.split(@options[:level_separator])
|
68
|
+
star_count = 1
|
69
|
+
target = find_target headings.reverse, star_count, 0, @filedata.length
|
70
|
+
data_string = @options[:data_format].call data
|
71
|
+
@filedata.insert target, data_string
|
72
|
+
end
|
73
|
+
|
74
|
+
def complete
|
75
|
+
@fd.seek 0
|
76
|
+
@fd.write filedata
|
77
|
+
@fd.close
|
78
|
+
end
|
79
|
+
|
80
|
+
private
|
81
|
+
|
82
|
+
def open_file file
|
83
|
+
if File.exist? file
|
84
|
+
@fd = File.open(file, 'r+')
|
85
|
+
@fd.seek 0
|
86
|
+
@filedata = @fd.read nil
|
87
|
+
else
|
88
|
+
@fd = File.open(file, 'w')
|
89
|
+
@filedata = ''
|
90
|
+
end
|
91
|
+
@filedata = @filedata.split(/[\n\r]/)
|
92
|
+
end
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
@@ -3,8 +3,32 @@ require 'active_resource'
|
|
3
3
|
module MessengerPigeon
|
4
4
|
# Redmine source/target module
|
5
5
|
module Redmine
|
6
|
+
# Module mixin for redmine activeresource init
|
7
|
+
module Init
|
8
|
+
@@known_instances = {}
|
9
|
+
|
10
|
+
def initialize(options)
|
11
|
+
@options = options
|
12
|
+
o = get_resource options[:resource]
|
13
|
+
o.site = options[:site]
|
14
|
+
o.user = options[:user]
|
15
|
+
o.password = options[:password]
|
16
|
+
o.include_root_in_json = true
|
17
|
+
o.format = JsonFormatter.new
|
18
|
+
@resource = o
|
19
|
+
end
|
20
|
+
|
21
|
+
private
|
22
|
+
|
23
|
+
def get_resource(name)
|
24
|
+
return @@known_instances[name] unless @@known_instances[name].nil?
|
25
|
+
ar_cl = Class.new ActiveResource::Base
|
26
|
+
@@known_instances[name] = Object.const_set(name, ar_cl)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
6
30
|
# Handle JSON returned by Redmine
|
7
|
-
class
|
31
|
+
class JsonFormatter
|
8
32
|
include ActiveResource::Formats::JsonFormat
|
9
33
|
|
10
34
|
def decode(json)
|
@@ -23,55 +47,43 @@ module MessengerPigeon
|
|
23
47
|
|
24
48
|
# Source definition
|
25
49
|
class Source
|
26
|
-
|
27
|
-
@options = options
|
28
|
-
ar_cl = Class.new ActiveResource::Base
|
29
|
-
o = Object.const_set(options[:resource], ar_cl)
|
30
|
-
o.site = options[:site]
|
31
|
-
o.user = options[:user]
|
32
|
-
o.password = options[:password]
|
33
|
-
o.include_root_in_json = true
|
34
|
-
o.format = RedmineFormatter.new
|
35
|
-
@resource = o
|
36
|
-
end
|
50
|
+
include Init
|
37
51
|
|
38
52
|
def read
|
39
53
|
if @options[:mode] == :specific
|
40
54
|
[@resource.find(@options[:key]).attributes]
|
41
55
|
elsif @options[:mode] == :all
|
42
|
-
|
43
|
-
res.map do |r|
|
44
|
-
attrs = r.attributes
|
45
|
-
attrs.each do |k, v|
|
46
|
-
attrs[k] = v.attributes if v.respond_to? :attributes
|
47
|
-
end
|
48
|
-
end
|
56
|
+
read_all
|
49
57
|
end
|
50
58
|
end
|
51
59
|
|
52
60
|
def complete
|
53
61
|
end
|
62
|
+
|
63
|
+
private
|
64
|
+
|
65
|
+
def read_all
|
66
|
+
res = @resource.find(:all, params: @options[:params])
|
67
|
+
res.map do |r|
|
68
|
+
attrs = r.attributes
|
69
|
+
attrs.each do |k, v|
|
70
|
+
# Convert 1 level of nested attributes
|
71
|
+
attrs[k] = v.attributes if v.respond_to? :attributes
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
54
75
|
end
|
55
76
|
|
56
77
|
# Target definition
|
57
78
|
class Target
|
58
|
-
|
59
|
-
@options = options
|
60
|
-
ar_cl = Class.new ActiveResource::Base
|
61
|
-
o = Object.const_set(options[:resource], ar_cl)
|
62
|
-
o.site = options[:site]
|
63
|
-
o.user = options[:user]
|
64
|
-
o.password = options[:password]
|
65
|
-
o.include_root_in_json = true
|
66
|
-
@resource = o
|
67
|
-
end
|
79
|
+
include Init
|
68
80
|
|
69
81
|
def update(data)
|
70
82
|
m = @resource.new data
|
71
83
|
$stderr.puts issue.errors.full_messages unless m.save
|
72
84
|
end
|
73
85
|
|
74
|
-
def
|
86
|
+
def complete
|
75
87
|
end
|
76
88
|
end
|
77
89
|
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
require 'sequel'
|
2
|
+
|
3
|
+
module MessengerPigeon
|
4
|
+
# Redmine source/target module
|
5
|
+
module SQL
|
6
|
+
# Source definition
|
7
|
+
module Init
|
8
|
+
attr_accessor :conn
|
9
|
+
|
10
|
+
def initialize(options)
|
11
|
+
@options = options
|
12
|
+
cs = options[:connection_string]
|
13
|
+
@conn = Sequel.connect(cs)
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
class Source
|
18
|
+
include Init
|
19
|
+
|
20
|
+
def read
|
21
|
+
@conn.fetch @options[:query]
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
# Module mixin for redmine activeresource init
|
26
|
+
class Target
|
27
|
+
include Init
|
28
|
+
|
29
|
+
def update(data)
|
30
|
+
tbl = @conn[@options[:table].intern]
|
31
|
+
data.each do |row|
|
32
|
+
tbl.insert row
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def complete
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
@@ -8,32 +8,33 @@ module MessengerPigeon
|
|
8
8
|
@filters = pigeon_config[:filters]
|
9
9
|
@transforms = pigeon_config[:transforms]
|
10
10
|
@generators = pigeon_config[:generators]
|
11
|
+
@logger = pigeon_config[:logger]
|
11
12
|
end
|
12
13
|
|
13
14
|
def fly
|
14
15
|
@source.read.each do |d|
|
15
16
|
transforms :pre_filter, d
|
16
17
|
generators d
|
17
|
-
|
18
|
+
if filter d
|
19
|
+
d = transforms :post_filter, d
|
20
|
+
@target.update d
|
21
|
+
@logger.call d if @logger
|
22
|
+
end
|
18
23
|
end
|
19
24
|
self
|
20
25
|
end
|
21
26
|
|
22
|
-
def finalise
|
23
|
-
@target.write
|
24
|
-
@source.complete
|
25
|
-
end
|
26
|
-
|
27
27
|
private
|
28
28
|
|
29
29
|
def filter(data)
|
30
30
|
return if @filters.nil?
|
31
31
|
@filters.each do |k, v|
|
32
|
-
|
32
|
+
case v
|
33
|
+
when Regexp
|
33
34
|
return false unless data[k] =~ v
|
34
|
-
|
35
|
+
when String
|
35
36
|
return false unless data[k] == v
|
36
|
-
|
37
|
+
when Proc
|
37
38
|
return false unless v.call data[k]
|
38
39
|
end
|
39
40
|
end
|
@@ -41,7 +42,7 @@ module MessengerPigeon
|
|
41
42
|
end
|
42
43
|
|
43
44
|
def transforms(type, data)
|
44
|
-
return data if @transforms.nil? || @transforms[
|
45
|
+
return data if @transforms.nil? || @transforms[type].nil?
|
45
46
|
@transforms[type].each do |k, v|
|
46
47
|
data[k] = v.call data[k]
|
47
48
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: messenger_pigeon
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Chris Mann
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-09-27 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -42,19 +42,48 @@ dependencies:
|
|
42
42
|
name: activeresource
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
|
-
- -
|
45
|
+
- - '='
|
46
46
|
- !ruby/object:Gem::Version
|
47
47
|
version: '4.0'
|
48
48
|
type: :development
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
|
-
- -
|
52
|
+
- - '='
|
53
53
|
- !ruby/object:Gem::Version
|
54
54
|
version: '4.0'
|
55
|
-
|
56
|
-
|
57
|
-
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: sequel
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '4.26'
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '4.26'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: sqlite3
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - "~>"
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '1.3'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - "~>"
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '1.3'
|
83
|
+
description: The goal of MessengerPigeon is to provide a highly-configurable and adaptable
|
84
|
+
animal to take record-based data from any number of sources, and copy that data
|
85
|
+
to any number of destinations. A pigeon may modify, drop or add to your data while
|
86
|
+
in transit, but should only do these things when you ask it nicely.
|
58
87
|
email:
|
59
88
|
- chris@bitpattern.com.au
|
60
89
|
executables:
|
@@ -68,11 +97,13 @@ files:
|
|
68
97
|
- lib/messenger_pigeon.rb
|
69
98
|
- lib/messenger_pigeon/cli.rb
|
70
99
|
- lib/messenger_pigeon/config.rb
|
71
|
-
- lib/messenger_pigeon/
|
72
|
-
- lib/messenger_pigeon/
|
73
|
-
- lib/messenger_pigeon/
|
100
|
+
- lib/messenger_pigeon/endpoint_manager.rb
|
101
|
+
- lib/messenger_pigeon/modules/console.rb
|
102
|
+
- lib/messenger_pigeon/modules/csv.rb
|
103
|
+
- lib/messenger_pigeon/modules/orgmode.rb
|
104
|
+
- lib/messenger_pigeon/modules/redmine.rb
|
105
|
+
- lib/messenger_pigeon/modules/sql.rb
|
74
106
|
- lib/messenger_pigeon/pigeon.rb
|
75
|
-
- lib/messenger_pigeon/redmine.rb
|
76
107
|
- lib/messenger_pigeon/version.rb
|
77
108
|
homepage: https://github.com/cshclm/MessengerPigeon
|
78
109
|
licenses:
|
@@ -86,7 +117,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
86
117
|
requirements:
|
87
118
|
- - ">="
|
88
119
|
- !ruby/object:Gem::Version
|
89
|
-
version:
|
120
|
+
version: 1.9.3
|
90
121
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
91
122
|
requirements:
|
92
123
|
- - ">="
|
@@ -1,56 +0,0 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
module MessengerPigeon
|
4
|
-
# OrgMode source/target module
|
5
|
-
module OrgMode
|
6
|
-
# Source definition
|
7
|
-
class Source
|
8
|
-
def initialize(_options = {})
|
9
|
-
fail 'Not Implemented'
|
10
|
-
end
|
11
|
-
end
|
12
|
-
|
13
|
-
# Target definition
|
14
|
-
class Target
|
15
|
-
def initialize(options = nil)
|
16
|
-
@options = options
|
17
|
-
if File.exist? options[:file]
|
18
|
-
@fd = File.open(options[:file], 'r+')
|
19
|
-
@fd.seek 0
|
20
|
-
@filedata = @fd.read nil
|
21
|
-
else
|
22
|
-
@fd = File.open(options[:file], 'w')
|
23
|
-
@filedata = ''
|
24
|
-
end
|
25
|
-
@known_headings = known_headings
|
26
|
-
end
|
27
|
-
|
28
|
-
def known_headings
|
29
|
-
res = []
|
30
|
-
@filedata.each_line do |l|
|
31
|
-
/^\*+\ (?<heading>.+?)(:[^ ]*:)?$/ =~ l
|
32
|
-
res.push heading if heading
|
33
|
-
end
|
34
|
-
res
|
35
|
-
end
|
36
|
-
|
37
|
-
def update(data)
|
38
|
-
heading = @options[:refile_target] % data
|
39
|
-
data_string = @options[:data_format].call data
|
40
|
-
if @known_headings.include? heading
|
41
|
-
@filedata.sub!(/^(\* #{heading}.*)$/,
|
42
|
-
"\\1\n#{data_string}")
|
43
|
-
else
|
44
|
-
@filedata += "* #{heading}\n#{data_string}\n"
|
45
|
-
@known_headings.push heading
|
46
|
-
end
|
47
|
-
end
|
48
|
-
|
49
|
-
def write
|
50
|
-
@fd.seek 0
|
51
|
-
@fd.write @filedata
|
52
|
-
@fd.close
|
53
|
-
end
|
54
|
-
end
|
55
|
-
end
|
56
|
-
end
|