issuesrc 0.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +15 -0
- data/.gitignore +14 -0
- data/Gemfile +9 -0
- data/LICENSE.txt +339 -0
- data/README.md +33 -0
- data/Rakefile +14 -0
- data/bin/issuesrc +93 -0
- data/example.toml +56 -0
- data/issuesrc.gemspec +28 -0
- data/lib/issuers/github_issuer.rb +247 -0
- data/lib/issuesrc/config.rb +42 -0
- data/lib/issuesrc/event_loop.rb +91 -0
- data/lib/issuesrc/file.rb +40 -0
- data/lib/issuesrc/tag.rb +61 -0
- data/lib/issuesrc/tag_extractor.rb +58 -0
- data/lib/issuesrc/version.rb +3 -0
- data/lib/issuesrc.rb +211 -0
- data/lib/sourcers/git_sourcer.rb +238 -0
- data/lib/sourcers/github_sourcer.rb +26 -0
- data/lib/tag_finders/blunt_tag_finder.rb +186 -0
- data/spec/issuers/github_issuer_spec.rb +0 -0
- data/spec/issuesrc/config_spec.rb +0 -0
- data/spec/issuesrc/event_loop_spec.rb +0 -0
- data/spec/issuesrc/file_spec.rb +0 -0
- data/spec/issuesrc/tag_extractor_spec.rb +0 -0
- data/spec/issuesrc/tag_spec.rb +0 -0
- data/spec/issuesrc_spec.rb +81 -0
- data/spec/sourcers/git_sourcer_spec.rb +0 -0
- data/spec/sourcers/github_sourcer_spec.rb +0 -0
- data/spec/tag_finders/blunt_tag_finder_spec.rb +0 -0
- metadata +155 -0
data/lib/issuesrc.rb
ADDED
@@ -0,0 +1,211 @@
|
|
1
|
+
# This program is free software: you can redistribute it and/or modify
|
2
|
+
# it under the terms of the GNU General Public License as published by
|
3
|
+
# the Free Software Foundation, either version 2 of the License, or
|
4
|
+
# (at your option) any later version.
|
5
|
+
|
6
|
+
require 'issuesrc/version'
|
7
|
+
require 'issuesrc/config'
|
8
|
+
require 'issuesrc/tag_extractor'
|
9
|
+
require 'issuesrc/event_loop'
|
10
|
+
|
11
|
+
module Issuesrc
|
12
|
+
DEFAULT_SOURCER = 'github'
|
13
|
+
DEFAULT_ISSUER = 'github'
|
14
|
+
DEFAULT_TAG_FINDERS = ['blunt']
|
15
|
+
|
16
|
+
SOURCERS = {
|
17
|
+
'git' => ['sourcers/git_sourcer', 'GitSourcer'],
|
18
|
+
'github' => ['sourcers/github_sourcer', 'GithubSourcer'],
|
19
|
+
}
|
20
|
+
|
21
|
+
ISSUERS = {
|
22
|
+
'github' => ['issuers/github_issuer', 'GithubIssuer'],
|
23
|
+
}
|
24
|
+
|
25
|
+
TAG_FINDERS = {
|
26
|
+
'blunt' => ['tag_finders/blunt_tag_finder', 'BluntTagFinder'],
|
27
|
+
}
|
28
|
+
|
29
|
+
# Run issuesrc.
|
30
|
+
def self.run(args, config)
|
31
|
+
program = Class.new { include Issuesrc }.new
|
32
|
+
program.set_config(args, config)
|
33
|
+
program.init_files_offsets()
|
34
|
+
|
35
|
+
event_loop = Issuesrc::SequentialEventLoop.new()
|
36
|
+
sourcer = program.load_sourcer()
|
37
|
+
tag_finders = program.load_tag_finders()
|
38
|
+
issuer = program.load_issuer(event_loop)
|
39
|
+
|
40
|
+
issues = issuer.async_load_issues()
|
41
|
+
|
42
|
+
created_tags, updated_tags, closed_issues = [], [], []
|
43
|
+
|
44
|
+
sourcer.retrieve_files().each do |file|
|
45
|
+
if Issuesrc::Config::option_from_args(:verbose, args)
|
46
|
+
puts file.path
|
47
|
+
end
|
48
|
+
|
49
|
+
tag_finder = program.select_tag_finder_for(file, tag_finders)
|
50
|
+
if tag_finder.nil?
|
51
|
+
next
|
52
|
+
end
|
53
|
+
|
54
|
+
tags = []
|
55
|
+
tag_finder.find_tags(file) { |tag| tags << tag }
|
56
|
+
|
57
|
+
tags_by_issue_id, new_tags = program.classify_tags(tags, file)
|
58
|
+
|
59
|
+
new_tags.each do |tag|
|
60
|
+
created_tags << tag
|
61
|
+
issuer.async_create_issue(tag) do |tag|
|
62
|
+
program.save_tag_in_file(tag)
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
issuer.async_update_or_close_issues(issues, tags_by_issue_id) do
|
67
|
+
|issue_id, tag, action|
|
68
|
+
case action
|
69
|
+
when :updated
|
70
|
+
program.save_tag_in_file(tag)
|
71
|
+
updated_tags << tag
|
72
|
+
when :closed
|
73
|
+
closed_issues << issue_id
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
event_loop.wait_for_pending()
|
79
|
+
|
80
|
+
if sourcer.respond_to? :finish
|
81
|
+
sourcer.finish(created_tags, updated_tags, closed_issues)
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
def set_config(args, config)
|
86
|
+
@args = args
|
87
|
+
@config = config
|
88
|
+
end
|
89
|
+
|
90
|
+
def init_files_offsets
|
91
|
+
@files_offsets = {}
|
92
|
+
end
|
93
|
+
|
94
|
+
# Creates the instance of the sourcer that should be used for the current
|
95
|
+
# execution of issuesrc. It looks first at the :sourcer command line
|
96
|
+
# argument, then `[sourcer] sourcer = ...` from the config file. If those are
|
97
|
+
# not present, `DEFAULT_SOURCER` will be used. If the selected sourcer is not
|
98
|
+
# implemented, ie. is not a key of `SOURCERS`, the execution will fail.
|
99
|
+
def load_sourcer
|
100
|
+
path, cls = load_component(
|
101
|
+
['sourcer', 'sourcer'],
|
102
|
+
:sourcer,
|
103
|
+
DEFAULT_SOURCER,
|
104
|
+
SOURCERS)
|
105
|
+
do_require(path)
|
106
|
+
make_sourcer(cls)
|
107
|
+
end
|
108
|
+
|
109
|
+
def make_sourcer(cls)
|
110
|
+
Issuesrc::Sourcers.const_get(cls).new(@args, @config)
|
111
|
+
end
|
112
|
+
|
113
|
+
# Like `load_sourcer`, but for the issuer. It first looks at :issuer from
|
114
|
+
# the command line arguments, then `[issuer] issuer = ...` from the config
|
115
|
+
# file.
|
116
|
+
def load_issuer(event_loop)
|
117
|
+
path, cls = load_component(
|
118
|
+
['issuer', 'issuer'],
|
119
|
+
:issuer,
|
120
|
+
DEFAULT_ISSUER,
|
121
|
+
ISSUERS)
|
122
|
+
do_require(path)
|
123
|
+
make_issuer(cls, event_loop)
|
124
|
+
end
|
125
|
+
|
126
|
+
def make_issuer(cls, event_loop)
|
127
|
+
Issuesrc::Issuers.const_get(cls).new(@args, @config, event_loop)
|
128
|
+
end
|
129
|
+
|
130
|
+
def load_component(config_key, arg_key, default, options)
|
131
|
+
type = Config::option_from_both(arg_key, config_key, @args, @config)
|
132
|
+
if type.nil?
|
133
|
+
type = default
|
134
|
+
end
|
135
|
+
load_component_by_type(type, options)
|
136
|
+
end
|
137
|
+
|
138
|
+
def load_component_by_type(type, options)
|
139
|
+
if !options.include?(type)
|
140
|
+
exec_fail 'Unrecognized sourcer type: #{type}'
|
141
|
+
end
|
142
|
+
|
143
|
+
options[type]
|
144
|
+
end
|
145
|
+
|
146
|
+
# Like `load_sourcer` but for the tag finders. It only looks at
|
147
|
+
# `[tag_finders] tag_finders = [...]` from the config file.
|
148
|
+
def load_tag_finders
|
149
|
+
tag_finders = Config::option_from_config(
|
150
|
+
['tag_finders', 'tag_finders'], @config)
|
151
|
+
if tag_finders.nil?
|
152
|
+
tag_finders = DEFAULT_TAG_FINDERS
|
153
|
+
end
|
154
|
+
load_tag_finders_by_types(tag_finders)
|
155
|
+
end
|
156
|
+
|
157
|
+
def load_tag_finders_by_types(types)
|
158
|
+
tag_extractor = Issuesrc::TagExtractor.new(@args, @config)
|
159
|
+
tag_finders = []
|
160
|
+
types.each do |type|
|
161
|
+
path, cls = load_component_by_type(type, TAG_FINDERS)
|
162
|
+
do_require(path)
|
163
|
+
tag_finders << make_tag_finder(cls, tag_extractor)
|
164
|
+
end
|
165
|
+
tag_finders
|
166
|
+
end
|
167
|
+
|
168
|
+
def make_tag_finder(cls, tag_extractor)
|
169
|
+
Issuesrc::TagFinders.const_get(cls).new(tag_extractor, @args, @config)
|
170
|
+
end
|
171
|
+
|
172
|
+
def select_tag_finder_for(file, tag_finders)
|
173
|
+
ret = nil
|
174
|
+
tag_finders.each do |tag_finder|
|
175
|
+
if tag_finder.accepts? file
|
176
|
+
ret = tag_finder
|
177
|
+
break
|
178
|
+
end
|
179
|
+
end
|
180
|
+
ret
|
181
|
+
end
|
182
|
+
|
183
|
+
def classify_tags(tags, file)
|
184
|
+
tags_by_issue = {}
|
185
|
+
new_tags = []
|
186
|
+
tags.each do |tag|
|
187
|
+
if tag.issue_id.nil?
|
188
|
+
new_tags << tag
|
189
|
+
else
|
190
|
+
tags_by_issue[tag.issue_id] = tag
|
191
|
+
end
|
192
|
+
end
|
193
|
+
[tags_by_issue, new_tags]
|
194
|
+
end
|
195
|
+
|
196
|
+
def save_tag_in_file(tag)
|
197
|
+
offsets = @files_offsets.fetch(tag.file.path, [])
|
198
|
+
offsets = tag.write_in_file(offsets)
|
199
|
+
@files_offsets[tag.file.path] = offsets
|
200
|
+
end
|
201
|
+
|
202
|
+
def self.exec_fail(feedback)
|
203
|
+
raise IssuesrcError, feedback
|
204
|
+
end
|
205
|
+
|
206
|
+
def do_require(path)
|
207
|
+
require path
|
208
|
+
end
|
209
|
+
|
210
|
+
class IssuesrcError < Exception; end
|
211
|
+
end
|
@@ -0,0 +1,238 @@
|
|
1
|
+
require 'tmpdir'
|
2
|
+
require 'open3'
|
3
|
+
require 'find'
|
4
|
+
require 'issuesrc/file'
|
5
|
+
require 'issuesrc/config'
|
6
|
+
require 'ptools'
|
7
|
+
|
8
|
+
module Issuesrc
|
9
|
+
module Sourcers
|
10
|
+
class GitSourcer
|
11
|
+
def initialize(args, config)
|
12
|
+
if @url.nil?
|
13
|
+
@url = try_find_repo_url(args, config)
|
14
|
+
end
|
15
|
+
|
16
|
+
if @url.nil?
|
17
|
+
@path = try_find_repo_path(args, config)
|
18
|
+
end
|
19
|
+
|
20
|
+
init(args, config)
|
21
|
+
end
|
22
|
+
|
23
|
+
def initialize_with_url(url, args, config)
|
24
|
+
@url = url
|
25
|
+
init(args, config)
|
26
|
+
end
|
27
|
+
|
28
|
+
def initialize_with_path(path, args, config)
|
29
|
+
@path = path
|
30
|
+
init(args, config)
|
31
|
+
end
|
32
|
+
|
33
|
+
def retrieve_files()
|
34
|
+
dir = nil
|
35
|
+
if !@url.nil?
|
36
|
+
tmp_dir = Dir::mktmpdir
|
37
|
+
@tmp_dir = tmp_dir
|
38
|
+
dir = clone_repo(tmp_dir)
|
39
|
+
@path = dir
|
40
|
+
else
|
41
|
+
dir = @path
|
42
|
+
end
|
43
|
+
|
44
|
+
Enumerator.new do |enum|
|
45
|
+
find_all_files(dir) do |file|
|
46
|
+
enum << file
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
def finish(created_tags, updated_tags, closed_issues)
|
52
|
+
if created_tags.empty? && closed_issues.empty?
|
53
|
+
return
|
54
|
+
end
|
55
|
+
|
56
|
+
msg = make_commit_message(created_tags, updated_tags, closed_issues)
|
57
|
+
$stderr.puts msg
|
58
|
+
|
59
|
+
if @commit_when_done
|
60
|
+
make_commit(msg)
|
61
|
+
end
|
62
|
+
|
63
|
+
if @push_when_done
|
64
|
+
push_repo()
|
65
|
+
end
|
66
|
+
|
67
|
+
if @tmp_dir
|
68
|
+
FileUtils.remove_entry_secure @tmp_dir
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
private
|
73
|
+
def init(args, config)
|
74
|
+
init_exclude(config)
|
75
|
+
@commit_when_done = decide_commit_when_done(args, config)
|
76
|
+
@push_when_done = decide_push_when_done(args, config)
|
77
|
+
end
|
78
|
+
|
79
|
+
def init_exclude(config)
|
80
|
+
@exclude = Issuesrc::Config::option_from_config(
|
81
|
+
['sourcer', 'exclude_files'], config)
|
82
|
+
if @exclude.nil?
|
83
|
+
@exclude = []
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
|
88
|
+
def try_find_repo_url(args, config)
|
89
|
+
Issuesrc::Config::option_from_both(:repo_url, ['git', 'repo'],
|
90
|
+
args, config)
|
91
|
+
end
|
92
|
+
|
93
|
+
def try_find_repo_path(args, config)
|
94
|
+
Issuesrc::Config::option_from_both(:repo_path, ['git', 'repo_path'],
|
95
|
+
args, config)
|
96
|
+
end
|
97
|
+
|
98
|
+
def decide_commit_when_done(args, config)
|
99
|
+
decide_when_done(:commit_when_done, 'commit_when_done', args, config)
|
100
|
+
end
|
101
|
+
|
102
|
+
def decide_push_when_done(args, config)
|
103
|
+
decide_when_done(:push_when_done, 'push_when_done', args, config)
|
104
|
+
end
|
105
|
+
|
106
|
+
def decide_when_done(args_key, config_key, args, config)
|
107
|
+
opt = Issuesrc::Config::option_from_both(
|
108
|
+
args_key, ['git', config_key], args, config)
|
109
|
+
if opt.nil?
|
110
|
+
opt = is_downloaded?
|
111
|
+
end
|
112
|
+
opt
|
113
|
+
end
|
114
|
+
|
115
|
+
def is_downloaded?
|
116
|
+
!@url.nil?
|
117
|
+
end
|
118
|
+
|
119
|
+
def clone_repo(dir)
|
120
|
+
out, err, status = Open3.capture3 "git clone #{@url} #{dir}/repo"
|
121
|
+
if status != 0
|
122
|
+
raise err
|
123
|
+
end
|
124
|
+
repo_dir = dir + '/repo'
|
125
|
+
change_branch_in_clone(repo_dir)
|
126
|
+
repo_dir
|
127
|
+
end
|
128
|
+
|
129
|
+
def find_all_files(dir)
|
130
|
+
set_branch_from_clone(dir)
|
131
|
+
Find.find(dir) do |path|
|
132
|
+
if FileTest.directory?(path) || File.binary?(path)
|
133
|
+
if File.basename(path) == '.git'
|
134
|
+
Find.prune
|
135
|
+
end
|
136
|
+
next
|
137
|
+
end
|
138
|
+
|
139
|
+
excluded = false
|
140
|
+
@exclude.each do |exc|
|
141
|
+
if File.fnmatch?(exc, path_in_repo(path))
|
142
|
+
excluded = true
|
143
|
+
next
|
144
|
+
end
|
145
|
+
end
|
146
|
+
if excluded
|
147
|
+
next
|
148
|
+
end
|
149
|
+
|
150
|
+
yield Issuesrc::GitFile.new(path, path_in_repo(path), @branch)
|
151
|
+
end
|
152
|
+
end
|
153
|
+
|
154
|
+
def path_in_repo(path)
|
155
|
+
path[@path.length + 1..-1]
|
156
|
+
end
|
157
|
+
|
158
|
+
def change_branch_in_clone(repo_dir)
|
159
|
+
if @branch.nil?
|
160
|
+
return
|
161
|
+
end
|
162
|
+
|
163
|
+
pwd = Dir::pwd
|
164
|
+
Dir::chdir(repo_dir)
|
165
|
+
out, err, status = Open3.capture3 "git checkout #{@branch}"
|
166
|
+
if status != 0
|
167
|
+
Dir::chdir(pwd)
|
168
|
+
raise err
|
169
|
+
end
|
170
|
+
Dir::chdir(pwd)
|
171
|
+
end
|
172
|
+
|
173
|
+
def set_branch_from_clone(repo_dir)
|
174
|
+
pwd = Dir::pwd
|
175
|
+
Dir::chdir(repo_dir)
|
176
|
+
out, err, status = Open3.capture3 "git rev-parse --abbrev-ref HEAD"
|
177
|
+
if status != 0
|
178
|
+
Dir::chdir(pwd)
|
179
|
+
raise err
|
180
|
+
end
|
181
|
+
@branch = out[0..-2]
|
182
|
+
Dir::chdir(pwd)
|
183
|
+
end
|
184
|
+
|
185
|
+
def make_commit_message(created_tags, updated_tags, closed_issues)
|
186
|
+
s = "Issuesrc: Synchronize issues from source code."
|
187
|
+
created_issues = created_tags.map { |x| x.issue_id }
|
188
|
+
s << make_commit_message_issues("Opens", created_issues)
|
189
|
+
s << make_commit_message_issues("Fixes", closed_issues)
|
190
|
+
end
|
191
|
+
|
192
|
+
def make_commit_message_issues(action, issue_ids)
|
193
|
+
if issue_ids.empty?
|
194
|
+
return ""
|
195
|
+
end
|
196
|
+
|
197
|
+
s = "\n\n"
|
198
|
+
parts = []
|
199
|
+
issue_ids.each do |issue_id|
|
200
|
+
parts << "#{action} \##{issue_id}"
|
201
|
+
end
|
202
|
+
s << parts.join("\n")
|
203
|
+
s
|
204
|
+
end
|
205
|
+
|
206
|
+
def make_commit(msg)
|
207
|
+
prev_dir = Dir::pwd
|
208
|
+
Dir::chdir(@path)
|
209
|
+
Open3.popen3("git commit -a --file -") do |fin, fout, ferr, proc|
|
210
|
+
fin.write(msg + "\n")
|
211
|
+
fin.close
|
212
|
+
out = fout.read
|
213
|
+
err = ferr.read
|
214
|
+
status = proc.value
|
215
|
+
fout.close
|
216
|
+
ferr.close
|
217
|
+
if status.to_i != 0
|
218
|
+
Dir::chdir(prev_dir)
|
219
|
+
puts "error committing: #{out} #{err}"
|
220
|
+
next
|
221
|
+
end
|
222
|
+
end
|
223
|
+
Dir::chdir(prev_dir)
|
224
|
+
end
|
225
|
+
|
226
|
+
def push_repo
|
227
|
+
prev_dir = Dir::pwd
|
228
|
+
Dir::chdir(@path)
|
229
|
+
out, err, status = Open3.capture3 "git push"
|
230
|
+
if status != 0
|
231
|
+
Dir::chdir(prev_dir)
|
232
|
+
raise err
|
233
|
+
end
|
234
|
+
Dir::chdir(prev_dir)
|
235
|
+
end
|
236
|
+
end
|
237
|
+
end
|
238
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
require 'sourcers/git_sourcer'
|
2
|
+
require 'issuesrc/config'
|
3
|
+
|
4
|
+
module Issuesrc
|
5
|
+
module Sourcers
|
6
|
+
class GithubSourcer < GitSourcer
|
7
|
+
def initialize(args, config)
|
8
|
+
begin
|
9
|
+
user, repo = try_find_repo(args, config)
|
10
|
+
url = "git@github.com:#{user}/#{repo}.git"
|
11
|
+
GitSourcer.instance_method(:initialize_with_url).bind(self).call(
|
12
|
+
url, args, config)
|
13
|
+
rescue Exception => e
|
14
|
+
super args, config
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
private
|
19
|
+
def try_find_repo(args, config)
|
20
|
+
repo_arg = Issuesrc::Config::option_from_both(
|
21
|
+
:repo, ['github', 'repo'], args, config)
|
22
|
+
repo_arg.split('/')
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,186 @@
|
|
1
|
+
require 'issuesrc/tag'
|
2
|
+
|
3
|
+
module Issuesrc
|
4
|
+
module TagFinders
|
5
|
+
class BluntTagFinder
|
6
|
+
DEFAULT_COMMENT_MARKERS = [['//', "\n"], ['/*', '*/']]
|
7
|
+
DEFAULT_STRING_MARKERS = ['"', "'"]
|
8
|
+
|
9
|
+
COMMENTS_BY_LANG = {
|
10
|
+
'php' => [['//', "\n"], ['/*', '*/'], ['#', "\n"]],
|
11
|
+
'html' => [['<!--', '-->']],
|
12
|
+
'sql' => [['--', "\n"]],
|
13
|
+
'sh' => [['#', "\n"]],
|
14
|
+
'hs' => [['--', "\n"], ['{-', '-}']],
|
15
|
+
'py' => [['#', "\n"]],
|
16
|
+
'rb' => [['#', "\n"]],
|
17
|
+
'clj' => [[';', "\n"]],
|
18
|
+
'coffee' => [['#', "\n"]],
|
19
|
+
}
|
20
|
+
|
21
|
+
STRINGS_BY_LANG = {
|
22
|
+
'go' => ['"', "'", '`'],
|
23
|
+
'rs' => ['"'],
|
24
|
+
'hs' => ['"'],
|
25
|
+
}
|
26
|
+
|
27
|
+
def initialize(tag_extractor, args, config)
|
28
|
+
@tag_extractor = tag_extractor
|
29
|
+
end
|
30
|
+
|
31
|
+
def accepts?(file)
|
32
|
+
true
|
33
|
+
end
|
34
|
+
|
35
|
+
def find_tags(file)
|
36
|
+
find_comments(file) do |comment, nline, pos|
|
37
|
+
# A tag extractor extracts from a single line, whereas comments may
|
38
|
+
# span several lines.
|
39
|
+
nline_offset = 0
|
40
|
+
comment.split("\n").each do |line|
|
41
|
+
tag_data = @tag_extractor.extract(line)
|
42
|
+
if !tag_data.nil?
|
43
|
+
yield Issuesrc::Tag.new(
|
44
|
+
tag_data['type'],
|
45
|
+
tag_data['issue_id'],
|
46
|
+
tag_data['author'],
|
47
|
+
tag_data['title'],
|
48
|
+
file,
|
49
|
+
nline + nline_offset,
|
50
|
+
pos + tag_data['begin_pos'],
|
51
|
+
pos + tag_data['end_pos']
|
52
|
+
)
|
53
|
+
end
|
54
|
+
pos += line.length + 1
|
55
|
+
nline_offset += 1
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
private
|
61
|
+
def find_comments(file)
|
62
|
+
comment_markers, string_markers = decide_markers(file)
|
63
|
+
body = file.body.read.force_encoding('BINARY') # TODO(#26): Use less memory here.
|
64
|
+
comment_finder = CommentFinder.new(body, comment_markers, string_markers)
|
65
|
+
pos = 0
|
66
|
+
|
67
|
+
comment_finder.each do |comment, nline, pos|
|
68
|
+
yield comment, nline, pos
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
def decide_markers(file)
|
73
|
+
[
|
74
|
+
COMMENTS_BY_LANG.fetch(file.type, DEFAULT_COMMENT_MARKERS),
|
75
|
+
STRINGS_BY_LANG.fetch(file.type, DEFAULT_STRING_MARKERS),
|
76
|
+
]
|
77
|
+
end
|
78
|
+
|
79
|
+
class CommentFinder
|
80
|
+
def initialize(body, comment_markers, string_markers)
|
81
|
+
@body = body
|
82
|
+
@comment_markers = comment_markers
|
83
|
+
@string_markers = string_markers
|
84
|
+
|
85
|
+
@pos = 0
|
86
|
+
@nline = 1
|
87
|
+
end
|
88
|
+
|
89
|
+
def each
|
90
|
+
state = :init
|
91
|
+
while !@body.empty?
|
92
|
+
case state
|
93
|
+
when :init
|
94
|
+
consumed, state = consume_init()
|
95
|
+
@pos += consumed
|
96
|
+
when :string
|
97
|
+
consumed, state = consume_string()
|
98
|
+
@pos += consumed
|
99
|
+
when :comment
|
100
|
+
nline = @nline
|
101
|
+
lex, consumed, state = read_comment()
|
102
|
+
yield [lex, nline, @pos]
|
103
|
+
@pos += consumed
|
104
|
+
end
|
105
|
+
end
|
106
|
+
end
|
107
|
+
|
108
|
+
private
|
109
|
+
def consume_init
|
110
|
+
consumed = 0
|
111
|
+
next_state = nil
|
112
|
+
while !@body.empty?
|
113
|
+
next_state = peek_next_state()
|
114
|
+
if next_state != :init
|
115
|
+
break
|
116
|
+
end
|
117
|
+
consumed += consume_body(1)
|
118
|
+
end
|
119
|
+
[consumed, next_state]
|
120
|
+
end
|
121
|
+
|
122
|
+
def consume_string
|
123
|
+
lex, consumed, state = read_delimited(@string_markers)
|
124
|
+
[consumed, state]
|
125
|
+
end
|
126
|
+
|
127
|
+
def read_comment
|
128
|
+
read_delimited(@comment_markers)
|
129
|
+
end
|
130
|
+
|
131
|
+
def read_delimited(markers)
|
132
|
+
consumed = state_boundary(markers, :begin)
|
133
|
+
lex = @body[0...consumed]
|
134
|
+
consume_body(consumed)
|
135
|
+
|
136
|
+
consumed_end = state_boundary(markers, :end)
|
137
|
+
while !@body.empty? && consumed_end.nil?
|
138
|
+
lex << @body[0]
|
139
|
+
consumed += consume_body(1)
|
140
|
+
consumed_end = state_boundary(markers, :end)
|
141
|
+
end
|
142
|
+
|
143
|
+
if !consumed_end.nil?
|
144
|
+
lex << @body[0...consumed_end]
|
145
|
+
consumed += consume_body(consumed_end)
|
146
|
+
end
|
147
|
+
|
148
|
+
[lex, consumed, peek_next_state()]
|
149
|
+
end
|
150
|
+
|
151
|
+
def peek_next_state()
|
152
|
+
if !state_boundary(@comment_markers, :begin).nil?
|
153
|
+
:comment
|
154
|
+
elsif !state_boundary(@string_markers, :begin).nil?
|
155
|
+
:string
|
156
|
+
else
|
157
|
+
:init
|
158
|
+
end
|
159
|
+
end
|
160
|
+
|
161
|
+
def state_boundary(markers, begin_or_end)
|
162
|
+
if @body.nil?
|
163
|
+
nil
|
164
|
+
end
|
165
|
+
|
166
|
+
markers.each do |marker|
|
167
|
+
if marker.instance_of? Array
|
168
|
+
marker = marker[begin_or_end == :begin ? 0 : 1]
|
169
|
+
end
|
170
|
+
if @body.start_with? marker
|
171
|
+
return marker.length
|
172
|
+
end
|
173
|
+
end
|
174
|
+
nil
|
175
|
+
end
|
176
|
+
|
177
|
+
def consume_body(n)
|
178
|
+
l = @body.length
|
179
|
+
@nline += @body[0...n].count "\n"
|
180
|
+
@body = @body[n..-1] || ''
|
181
|
+
l - @body.length
|
182
|
+
end
|
183
|
+
end
|
184
|
+
end
|
185
|
+
end
|
186
|
+
end
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|