issuesrc 0.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/lib/issuesrc.rb ADDED
@@ -0,0 +1,211 @@
1
+ # This program is free software: you can redistribute it and/or modify
2
+ # it under the terms of the GNU General Public License as published by
3
+ # the Free Software Foundation, either version 2 of the License, or
4
+ # (at your option) any later version.
5
+
6
+ require 'issuesrc/version'
7
+ require 'issuesrc/config'
8
+ require 'issuesrc/tag_extractor'
9
+ require 'issuesrc/event_loop'
10
+
11
+ module Issuesrc
12
+ DEFAULT_SOURCER = 'github'
13
+ DEFAULT_ISSUER = 'github'
14
+ DEFAULT_TAG_FINDERS = ['blunt']
15
+
16
+ SOURCERS = {
17
+ 'git' => ['sourcers/git_sourcer', 'GitSourcer'],
18
+ 'github' => ['sourcers/github_sourcer', 'GithubSourcer'],
19
+ }
20
+
21
+ ISSUERS = {
22
+ 'github' => ['issuers/github_issuer', 'GithubIssuer'],
23
+ }
24
+
25
+ TAG_FINDERS = {
26
+ 'blunt' => ['tag_finders/blunt_tag_finder', 'BluntTagFinder'],
27
+ }
28
+
29
+ # Run issuesrc.
30
+ def self.run(args, config)
31
+ program = Class.new { include Issuesrc }.new
32
+ program.set_config(args, config)
33
+ program.init_files_offsets()
34
+
35
+ event_loop = Issuesrc::SequentialEventLoop.new()
36
+ sourcer = program.load_sourcer()
37
+ tag_finders = program.load_tag_finders()
38
+ issuer = program.load_issuer(event_loop)
39
+
40
+ issues = issuer.async_load_issues()
41
+
42
+ created_tags, updated_tags, closed_issues = [], [], []
43
+
44
+ sourcer.retrieve_files().each do |file|
45
+ if Issuesrc::Config::option_from_args(:verbose, args)
46
+ puts file.path
47
+ end
48
+
49
+ tag_finder = program.select_tag_finder_for(file, tag_finders)
50
+ if tag_finder.nil?
51
+ next
52
+ end
53
+
54
+ tags = []
55
+ tag_finder.find_tags(file) { |tag| tags << tag }
56
+
57
+ tags_by_issue_id, new_tags = program.classify_tags(tags, file)
58
+
59
+ new_tags.each do |tag|
60
+ created_tags << tag
61
+ issuer.async_create_issue(tag) do |tag|
62
+ program.save_tag_in_file(tag)
63
+ end
64
+ end
65
+
66
+ issuer.async_update_or_close_issues(issues, tags_by_issue_id) do
67
+ |issue_id, tag, action|
68
+ case action
69
+ when :updated
70
+ program.save_tag_in_file(tag)
71
+ updated_tags << tag
72
+ when :closed
73
+ closed_issues << issue_id
74
+ end
75
+ end
76
+ end
77
+
78
+ event_loop.wait_for_pending()
79
+
80
+ if sourcer.respond_to? :finish
81
+ sourcer.finish(created_tags, updated_tags, closed_issues)
82
+ end
83
+ end
84
+
85
+ def set_config(args, config)
86
+ @args = args
87
+ @config = config
88
+ end
89
+
90
+ def init_files_offsets
91
+ @files_offsets = {}
92
+ end
93
+
94
+ # Creates the instance of the sourcer that should be used for the current
95
+ # execution of issuesrc. It looks first at the :sourcer command line
96
+ # argument, then `[sourcer] sourcer = ...` from the config file. If those are
97
+ # not present, `DEFAULT_SOURCER` will be used. If the selected sourcer is not
98
+ # implemented, ie. is not a key of `SOURCERS`, the execution will fail.
99
+ def load_sourcer
100
+ path, cls = load_component(
101
+ ['sourcer', 'sourcer'],
102
+ :sourcer,
103
+ DEFAULT_SOURCER,
104
+ SOURCERS)
105
+ do_require(path)
106
+ make_sourcer(cls)
107
+ end
108
+
109
+ def make_sourcer(cls)
110
+ Issuesrc::Sourcers.const_get(cls).new(@args, @config)
111
+ end
112
+
113
+ # Like `load_sourcer`, but for the issuer. It first looks at :issuer from
114
+ # the command line arguments, then `[issuer] issuer = ...` from the config
115
+ # file.
116
+ def load_issuer(event_loop)
117
+ path, cls = load_component(
118
+ ['issuer', 'issuer'],
119
+ :issuer,
120
+ DEFAULT_ISSUER,
121
+ ISSUERS)
122
+ do_require(path)
123
+ make_issuer(cls, event_loop)
124
+ end
125
+
126
+ def make_issuer(cls, event_loop)
127
+ Issuesrc::Issuers.const_get(cls).new(@args, @config, event_loop)
128
+ end
129
+
130
+ def load_component(config_key, arg_key, default, options)
131
+ type = Config::option_from_both(arg_key, config_key, @args, @config)
132
+ if type.nil?
133
+ type = default
134
+ end
135
+ load_component_by_type(type, options)
136
+ end
137
+
138
+ def load_component_by_type(type, options)
139
+ if !options.include?(type)
140
+ exec_fail 'Unrecognized sourcer type: #{type}'
141
+ end
142
+
143
+ options[type]
144
+ end
145
+
146
+ # Like `load_sourcer` but for the tag finders. It only looks at
147
+ # `[tag_finders] tag_finders = [...]` from the config file.
148
+ def load_tag_finders
149
+ tag_finders = Config::option_from_config(
150
+ ['tag_finders', 'tag_finders'], @config)
151
+ if tag_finders.nil?
152
+ tag_finders = DEFAULT_TAG_FINDERS
153
+ end
154
+ load_tag_finders_by_types(tag_finders)
155
+ end
156
+
157
+ def load_tag_finders_by_types(types)
158
+ tag_extractor = Issuesrc::TagExtractor.new(@args, @config)
159
+ tag_finders = []
160
+ types.each do |type|
161
+ path, cls = load_component_by_type(type, TAG_FINDERS)
162
+ do_require(path)
163
+ tag_finders << make_tag_finder(cls, tag_extractor)
164
+ end
165
+ tag_finders
166
+ end
167
+
168
+ def make_tag_finder(cls, tag_extractor)
169
+ Issuesrc::TagFinders.const_get(cls).new(tag_extractor, @args, @config)
170
+ end
171
+
172
+ def select_tag_finder_for(file, tag_finders)
173
+ ret = nil
174
+ tag_finders.each do |tag_finder|
175
+ if tag_finder.accepts? file
176
+ ret = tag_finder
177
+ break
178
+ end
179
+ end
180
+ ret
181
+ end
182
+
183
+ def classify_tags(tags, file)
184
+ tags_by_issue = {}
185
+ new_tags = []
186
+ tags.each do |tag|
187
+ if tag.issue_id.nil?
188
+ new_tags << tag
189
+ else
190
+ tags_by_issue[tag.issue_id] = tag
191
+ end
192
+ end
193
+ [tags_by_issue, new_tags]
194
+ end
195
+
196
+ def save_tag_in_file(tag)
197
+ offsets = @files_offsets.fetch(tag.file.path, [])
198
+ offsets = tag.write_in_file(offsets)
199
+ @files_offsets[tag.file.path] = offsets
200
+ end
201
+
202
+ def self.exec_fail(feedback)
203
+ raise IssuesrcError, feedback
204
+ end
205
+
206
+ def do_require(path)
207
+ require path
208
+ end
209
+
210
+ class IssuesrcError < Exception; end
211
+ end
@@ -0,0 +1,238 @@
1
+ require 'tmpdir'
2
+ require 'open3'
3
+ require 'find'
4
+ require 'issuesrc/file'
5
+ require 'issuesrc/config'
6
+ require 'ptools'
7
+
8
+ module Issuesrc
9
+ module Sourcers
10
+ class GitSourcer
11
+ def initialize(args, config)
12
+ if @url.nil?
13
+ @url = try_find_repo_url(args, config)
14
+ end
15
+
16
+ if @url.nil?
17
+ @path = try_find_repo_path(args, config)
18
+ end
19
+
20
+ init(args, config)
21
+ end
22
+
23
+ def initialize_with_url(url, args, config)
24
+ @url = url
25
+ init(args, config)
26
+ end
27
+
28
+ def initialize_with_path(path, args, config)
29
+ @path = path
30
+ init(args, config)
31
+ end
32
+
33
+ def retrieve_files()
34
+ dir = nil
35
+ if !@url.nil?
36
+ tmp_dir = Dir::mktmpdir
37
+ @tmp_dir = tmp_dir
38
+ dir = clone_repo(tmp_dir)
39
+ @path = dir
40
+ else
41
+ dir = @path
42
+ end
43
+
44
+ Enumerator.new do |enum|
45
+ find_all_files(dir) do |file|
46
+ enum << file
47
+ end
48
+ end
49
+ end
50
+
51
+ def finish(created_tags, updated_tags, closed_issues)
52
+ if created_tags.empty? && closed_issues.empty?
53
+ return
54
+ end
55
+
56
+ msg = make_commit_message(created_tags, updated_tags, closed_issues)
57
+ $stderr.puts msg
58
+
59
+ if @commit_when_done
60
+ make_commit(msg)
61
+ end
62
+
63
+ if @push_when_done
64
+ push_repo()
65
+ end
66
+
67
+ if @tmp_dir
68
+ FileUtils.remove_entry_secure @tmp_dir
69
+ end
70
+ end
71
+
72
+ private
73
+ def init(args, config)
74
+ init_exclude(config)
75
+ @commit_when_done = decide_commit_when_done(args, config)
76
+ @push_when_done = decide_push_when_done(args, config)
77
+ end
78
+
79
+ def init_exclude(config)
80
+ @exclude = Issuesrc::Config::option_from_config(
81
+ ['sourcer', 'exclude_files'], config)
82
+ if @exclude.nil?
83
+ @exclude = []
84
+ end
85
+ end
86
+
87
+
88
+ def try_find_repo_url(args, config)
89
+ Issuesrc::Config::option_from_both(:repo_url, ['git', 'repo'],
90
+ args, config)
91
+ end
92
+
93
+ def try_find_repo_path(args, config)
94
+ Issuesrc::Config::option_from_both(:repo_path, ['git', 'repo_path'],
95
+ args, config)
96
+ end
97
+
98
+ def decide_commit_when_done(args, config)
99
+ decide_when_done(:commit_when_done, 'commit_when_done', args, config)
100
+ end
101
+
102
+ def decide_push_when_done(args, config)
103
+ decide_when_done(:push_when_done, 'push_when_done', args, config)
104
+ end
105
+
106
+ def decide_when_done(args_key, config_key, args, config)
107
+ opt = Issuesrc::Config::option_from_both(
108
+ args_key, ['git', config_key], args, config)
109
+ if opt.nil?
110
+ opt = is_downloaded?
111
+ end
112
+ opt
113
+ end
114
+
115
+ def is_downloaded?
116
+ !@url.nil?
117
+ end
118
+
119
+ def clone_repo(dir)
120
+ out, err, status = Open3.capture3 "git clone #{@url} #{dir}/repo"
121
+ if status != 0
122
+ raise err
123
+ end
124
+ repo_dir = dir + '/repo'
125
+ change_branch_in_clone(repo_dir)
126
+ repo_dir
127
+ end
128
+
129
+ def find_all_files(dir)
130
+ set_branch_from_clone(dir)
131
+ Find.find(dir) do |path|
132
+ if FileTest.directory?(path) || File.binary?(path)
133
+ if File.basename(path) == '.git'
134
+ Find.prune
135
+ end
136
+ next
137
+ end
138
+
139
+ excluded = false
140
+ @exclude.each do |exc|
141
+ if File.fnmatch?(exc, path_in_repo(path))
142
+ excluded = true
143
+ next
144
+ end
145
+ end
146
+ if excluded
147
+ next
148
+ end
149
+
150
+ yield Issuesrc::GitFile.new(path, path_in_repo(path), @branch)
151
+ end
152
+ end
153
+
154
+ def path_in_repo(path)
155
+ path[@path.length + 1..-1]
156
+ end
157
+
158
+ def change_branch_in_clone(repo_dir)
159
+ if @branch.nil?
160
+ return
161
+ end
162
+
163
+ pwd = Dir::pwd
164
+ Dir::chdir(repo_dir)
165
+ out, err, status = Open3.capture3 "git checkout #{@branch}"
166
+ if status != 0
167
+ Dir::chdir(pwd)
168
+ raise err
169
+ end
170
+ Dir::chdir(pwd)
171
+ end
172
+
173
+ def set_branch_from_clone(repo_dir)
174
+ pwd = Dir::pwd
175
+ Dir::chdir(repo_dir)
176
+ out, err, status = Open3.capture3 "git rev-parse --abbrev-ref HEAD"
177
+ if status != 0
178
+ Dir::chdir(pwd)
179
+ raise err
180
+ end
181
+ @branch = out[0..-2]
182
+ Dir::chdir(pwd)
183
+ end
184
+
185
+ def make_commit_message(created_tags, updated_tags, closed_issues)
186
+ s = "Issuesrc: Synchronize issues from source code."
187
+ created_issues = created_tags.map { |x| x.issue_id }
188
+ s << make_commit_message_issues("Opens", created_issues)
189
+ s << make_commit_message_issues("Fixes", closed_issues)
190
+ end
191
+
192
+ def make_commit_message_issues(action, issue_ids)
193
+ if issue_ids.empty?
194
+ return ""
195
+ end
196
+
197
+ s = "\n\n"
198
+ parts = []
199
+ issue_ids.each do |issue_id|
200
+ parts << "#{action} \##{issue_id}"
201
+ end
202
+ s << parts.join("\n")
203
+ s
204
+ end
205
+
206
+ def make_commit(msg)
207
+ prev_dir = Dir::pwd
208
+ Dir::chdir(@path)
209
+ Open3.popen3("git commit -a --file -") do |fin, fout, ferr, proc|
210
+ fin.write(msg + "\n")
211
+ fin.close
212
+ out = fout.read
213
+ err = ferr.read
214
+ status = proc.value
215
+ fout.close
216
+ ferr.close
217
+ if status.to_i != 0
218
+ Dir::chdir(prev_dir)
219
+ puts "error committing: #{out} #{err}"
220
+ next
221
+ end
222
+ end
223
+ Dir::chdir(prev_dir)
224
+ end
225
+
226
+ def push_repo
227
+ prev_dir = Dir::pwd
228
+ Dir::chdir(@path)
229
+ out, err, status = Open3.capture3 "git push"
230
+ if status != 0
231
+ Dir::chdir(prev_dir)
232
+ raise err
233
+ end
234
+ Dir::chdir(prev_dir)
235
+ end
236
+ end
237
+ end
238
+ end
@@ -0,0 +1,26 @@
1
+ require 'sourcers/git_sourcer'
2
+ require 'issuesrc/config'
3
+
4
+ module Issuesrc
5
+ module Sourcers
6
+ class GithubSourcer < GitSourcer
7
+ def initialize(args, config)
8
+ begin
9
+ user, repo = try_find_repo(args, config)
10
+ url = "git@github.com:#{user}/#{repo}.git"
11
+ GitSourcer.instance_method(:initialize_with_url).bind(self).call(
12
+ url, args, config)
13
+ rescue Exception => e
14
+ super args, config
15
+ end
16
+ end
17
+
18
+ private
19
+ def try_find_repo(args, config)
20
+ repo_arg = Issuesrc::Config::option_from_both(
21
+ :repo, ['github', 'repo'], args, config)
22
+ repo_arg.split('/')
23
+ end
24
+ end
25
+ end
26
+ end
@@ -0,0 +1,186 @@
1
+ require 'issuesrc/tag'
2
+
3
+ module Issuesrc
4
+ module TagFinders
5
+ class BluntTagFinder
6
+ DEFAULT_COMMENT_MARKERS = [['//', "\n"], ['/*', '*/']]
7
+ DEFAULT_STRING_MARKERS = ['"', "'"]
8
+
9
+ COMMENTS_BY_LANG = {
10
+ 'php' => [['//', "\n"], ['/*', '*/'], ['#', "\n"]],
11
+ 'html' => [['<!--', '-->']],
12
+ 'sql' => [['--', "\n"]],
13
+ 'sh' => [['#', "\n"]],
14
+ 'hs' => [['--', "\n"], ['{-', '-}']],
15
+ 'py' => [['#', "\n"]],
16
+ 'rb' => [['#', "\n"]],
17
+ 'clj' => [[';', "\n"]],
18
+ 'coffee' => [['#', "\n"]],
19
+ }
20
+
21
+ STRINGS_BY_LANG = {
22
+ 'go' => ['"', "'", '`'],
23
+ 'rs' => ['"'],
24
+ 'hs' => ['"'],
25
+ }
26
+
27
+ def initialize(tag_extractor, args, config)
28
+ @tag_extractor = tag_extractor
29
+ end
30
+
31
+ def accepts?(file)
32
+ true
33
+ end
34
+
35
+ def find_tags(file)
36
+ find_comments(file) do |comment, nline, pos|
37
+ # A tag extractor extracts from a single line, whereas comments may
38
+ # span several lines.
39
+ nline_offset = 0
40
+ comment.split("\n").each do |line|
41
+ tag_data = @tag_extractor.extract(line)
42
+ if !tag_data.nil?
43
+ yield Issuesrc::Tag.new(
44
+ tag_data['type'],
45
+ tag_data['issue_id'],
46
+ tag_data['author'],
47
+ tag_data['title'],
48
+ file,
49
+ nline + nline_offset,
50
+ pos + tag_data['begin_pos'],
51
+ pos + tag_data['end_pos']
52
+ )
53
+ end
54
+ pos += line.length + 1
55
+ nline_offset += 1
56
+ end
57
+ end
58
+ end
59
+
60
+ private
61
+ def find_comments(file)
62
+ comment_markers, string_markers = decide_markers(file)
63
+ body = file.body.read.force_encoding('BINARY') # TODO(#26): Use less memory here.
64
+ comment_finder = CommentFinder.new(body, comment_markers, string_markers)
65
+ pos = 0
66
+
67
+ comment_finder.each do |comment, nline, pos|
68
+ yield comment, nline, pos
69
+ end
70
+ end
71
+
72
+ def decide_markers(file)
73
+ [
74
+ COMMENTS_BY_LANG.fetch(file.type, DEFAULT_COMMENT_MARKERS),
75
+ STRINGS_BY_LANG.fetch(file.type, DEFAULT_STRING_MARKERS),
76
+ ]
77
+ end
78
+
79
+ class CommentFinder
80
+ def initialize(body, comment_markers, string_markers)
81
+ @body = body
82
+ @comment_markers = comment_markers
83
+ @string_markers = string_markers
84
+
85
+ @pos = 0
86
+ @nline = 1
87
+ end
88
+
89
+ def each
90
+ state = :init
91
+ while !@body.empty?
92
+ case state
93
+ when :init
94
+ consumed, state = consume_init()
95
+ @pos += consumed
96
+ when :string
97
+ consumed, state = consume_string()
98
+ @pos += consumed
99
+ when :comment
100
+ nline = @nline
101
+ lex, consumed, state = read_comment()
102
+ yield [lex, nline, @pos]
103
+ @pos += consumed
104
+ end
105
+ end
106
+ end
107
+
108
+ private
109
+ def consume_init
110
+ consumed = 0
111
+ next_state = nil
112
+ while !@body.empty?
113
+ next_state = peek_next_state()
114
+ if next_state != :init
115
+ break
116
+ end
117
+ consumed += consume_body(1)
118
+ end
119
+ [consumed, next_state]
120
+ end
121
+
122
+ def consume_string
123
+ lex, consumed, state = read_delimited(@string_markers)
124
+ [consumed, state]
125
+ end
126
+
127
+ def read_comment
128
+ read_delimited(@comment_markers)
129
+ end
130
+
131
+ def read_delimited(markers)
132
+ consumed = state_boundary(markers, :begin)
133
+ lex = @body[0...consumed]
134
+ consume_body(consumed)
135
+
136
+ consumed_end = state_boundary(markers, :end)
137
+ while !@body.empty? && consumed_end.nil?
138
+ lex << @body[0]
139
+ consumed += consume_body(1)
140
+ consumed_end = state_boundary(markers, :end)
141
+ end
142
+
143
+ if !consumed_end.nil?
144
+ lex << @body[0...consumed_end]
145
+ consumed += consume_body(consumed_end)
146
+ end
147
+
148
+ [lex, consumed, peek_next_state()]
149
+ end
150
+
151
+ def peek_next_state()
152
+ if !state_boundary(@comment_markers, :begin).nil?
153
+ :comment
154
+ elsif !state_boundary(@string_markers, :begin).nil?
155
+ :string
156
+ else
157
+ :init
158
+ end
159
+ end
160
+
161
+ def state_boundary(markers, begin_or_end)
162
+ if @body.nil?
163
+ nil
164
+ end
165
+
166
+ markers.each do |marker|
167
+ if marker.instance_of? Array
168
+ marker = marker[begin_or_end == :begin ? 0 : 1]
169
+ end
170
+ if @body.start_with? marker
171
+ return marker.length
172
+ end
173
+ end
174
+ nil
175
+ end
176
+
177
+ def consume_body(n)
178
+ l = @body.length
179
+ @nline += @body[0...n].count "\n"
180
+ @body = @body[n..-1] || ''
181
+ l - @body.length
182
+ end
183
+ end
184
+ end
185
+ end
186
+ end
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes