starscope 1.1.2 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/lib/starscope/db.rb CHANGED
@@ -3,103 +3,83 @@ require 'oj'
3
3
  require 'set'
4
4
  require 'zlib'
5
5
 
6
+ require 'starscope/export'
6
7
  require 'starscope/matcher'
7
8
  require 'starscope/output'
8
- require 'starscope/record'
9
-
10
- # cscope has this funky issue where it refuses to recognize function calls that
11
- # happen outside of a function definition - this isn't an issue in C, where all
12
- # calls must occur in a function, but in ruby et al. it is perfectly legal to
13
- # write normal code outside the "scope" of a function definition - we insert a
14
- # fake shim "global" function everywhere we can to work around this
15
- CSCOPE_GLOBAL_HACK_START = "\n\t$-\n"
16
- CSCOPE_GLOBAL_HACK_STOP = "\n\t}\n"
17
9
 
18
10
  # dynamically load all our language extractors
19
- LANGS = []
11
+ LANGS = {}
12
+ EXTRACTORS = []
20
13
  Dir.glob("#{File.dirname(__FILE__)}/langs/*.rb").each do |path|
21
14
  require path
22
- lang = /(\w+)\.rb$/.match(path)[1]
23
- LANGS << eval("StarScope::Lang::#{lang.capitalize}")
15
+ lang = /(\w+)\.rb$/.match(path)[1].capitalize
16
+ mod_name = "Starscope::Lang::#{lang}"
17
+ EXTRACTORS << eval(mod_name)
18
+ LANGS[lang.to_sym] = eval("#{mod_name}::VERSION")
24
19
  end
25
20
 
26
- class StarScope::DB
21
+ class Starscope::DB
22
+
23
+ include Starscope::Export
27
24
 
28
25
  DB_FORMAT = 5
29
26
 
30
27
  class NoTableError < StandardError; end
31
28
  class UnknownDBFormatError < StandardError; end
32
29
 
33
- def initialize(output_level)
34
- @output = StarScope::Output.new(output_level)
30
+ def initialize(output)
31
+ @output = output
35
32
  @meta = {:paths => [], :files => {}, :excludes => [],
36
- :version => StarScope::VERSION}
33
+ :langs => LANGS, :version => Starscope::VERSION}
37
34
  @tables = {}
38
35
  end
39
36
 
40
- # returns true if the database had to be up-converted from an old format
41
- def load(file)
42
- @output.log("Reading database from `#{file}`... ")
43
- File.open(file, 'r') do |file|
44
- Zlib::GzipReader.wrap(file) do |file|
45
- case file.gets.to_i
46
- when DB_FORMAT
47
- @meta = Oj.load(file.gets)
48
- @tables = Oj.load(file.gets)
49
- return false
50
- when 3..4
51
- # Old format, so read the directories segment then rebuild
52
- add_paths(Oj.load(file.gets))
53
- return true
54
- when 0..2
55
- # Old format (pre-json), so read the directories segment then rebuild
56
- len = file.gets.to_i
57
- add_paths(Marshal::load(file.read(len)))
58
- return true
59
- else
60
- raise UnknownDBFormatError
61
- end
62
- end
63
- end
37
+ def load(filename)
38
+ @output.extra("Reading database from `#{filename}`... ")
39
+ current_fmt = open_db(filename)
40
+ fixup if current_fmt
41
+ current_fmt
64
42
  end
65
43
 
66
- def save(file)
67
- @output.log("Writing database to `#{file}`...")
44
+ def save(filename)
45
+ @output.extra("Writing database to `#{filename}`...")
68
46
 
69
47
  # regardless of what the old version was, the new version is written by us
70
- @meta[:version] = StarScope::VERSION
48
+ @meta[:version] = Starscope::VERSION
49
+
50
+ @meta[:langs].merge!(LANGS)
71
51
 
72
- File.open(file, 'w') do |file|
73
- Zlib::GzipWriter.wrap(file) do |file|
74
- file.puts DB_FORMAT
75
- file.puts Oj.dump @meta
76
- file.puts Oj.dump @tables
52
+ File.open(filename, 'w') do |file|
53
+ Zlib::GzipWriter.wrap(file) do |stream|
54
+ stream.puts DB_FORMAT
55
+ stream.puts Oj.dump @meta
56
+ stream.puts Oj.dump @tables
77
57
  end
78
58
  end
79
59
  end
80
60
 
81
61
  def add_excludes(paths)
82
- @output.log("Excluding files in paths #{paths}...")
83
- @meta[:paths] -= paths.map {|p| normalize_glob(p)}
84
- paths = paths.map {|p| normalize_fnmatch(p)}
62
+ @output.extra("Excluding files in paths #{paths}...")
63
+ @meta[:paths] -= paths.map {|p| self.class.normalize_glob(p)}
64
+ paths = paths.map {|p| self.class.normalize_fnmatch(p)}
85
65
  @meta[:excludes] += paths
86
66
  @meta[:excludes].uniq!
87
67
 
88
- excluded = @meta[:files].keys.select {|name| matches_exclude?(paths, name)}
68
+ excluded = @meta[:files].keys.select {|name| matches_exclude?(name, paths)}
89
69
  remove_files(excluded)
90
70
  end
91
71
 
92
72
  def add_paths(paths)
93
- @output.log("Adding files in paths #{paths}...")
94
- @meta[:excludes] -= paths.map {|p| normalize_fnmatch(p)}
95
- paths = paths.map {|p| normalize_glob(p)}
73
+ @output.extra("Adding files in paths #{paths}...")
74
+ @meta[:excludes] -= paths.map {|p| self.class.normalize_fnmatch(p)}
75
+ paths = paths.map {|p| self.class.normalize_glob(p)}
96
76
  @meta[:paths] += paths
97
77
  @meta[:paths].uniq!
98
78
  files = Dir.glob(paths).select {|f| File.file? f}
99
- files.delete_if {|f| matches_exclude?(@meta[:excludes], f)}
79
+ files.delete_if {|f| matches_exclude?(f)}
100
80
  return if files.empty?
101
81
  @output.new_pbar("Building", files.length)
102
- add_new_files(files)
82
+ add_files(files)
103
83
  @output.finish_pbar
104
84
  end
105
85
 
@@ -109,189 +89,106 @@ class StarScope::DB
109
89
  changes[:deleted] ||= []
110
90
 
111
91
  new_files = (Dir.glob(@meta[:paths]).select {|f| File.file? f}) - @meta[:files].keys
112
- new_files.delete_if {|f| matches_exclude?(@meta[:excludes], f)}
92
+ new_files.delete_if {|f| matches_exclude?(f)}
113
93
 
114
94
  if changes[:deleted].empty? && changes[:modified].empty? && new_files.empty?
115
- @output.print("No changes detected.")
95
+ @output.normal("No changes detected.")
116
96
  return false
117
97
  end
118
98
 
119
99
  @output.new_pbar("Updating", changes[:modified].length + new_files.length)
120
100
  remove_files(changes[:deleted])
121
101
  update_files(changes[:modified])
122
- add_new_files(new_files)
102
+ add_files(new_files)
123
103
  @output.finish_pbar
124
104
 
125
105
  true
126
106
  end
127
107
 
128
- def dump_table(table)
108
+ def query(table, value)
129
109
  raise NoTableError if not @tables[table]
130
-
131
- puts "== Table: #{table} =="
132
- puts "No records" if @tables[table].empty?
133
-
134
- @tables[table].sort {|a,b|
135
- a[:name][-1].to_s.downcase <=> b[:name][-1].to_s.downcase
136
- }.each do |record|
137
- puts StarScope::Record.format(record)
138
- end
139
- end
140
-
141
- def dump_meta(key)
142
- if key == :meta
143
- puts "== Metadata Summary =="
144
- @meta.each do |k, v|
145
- print "#{k}: "
146
- if [Array, Hash].include? v.class
147
- puts v.count
148
- else
149
- puts v
150
- end
151
- end
152
- return
153
- end
154
- raise NoTableError if not @meta[key]
155
- puts "== Metadata: #{key} =="
156
- if @meta[key].is_a? Array
157
- @meta[key].sort.each {|x| puts x}
158
- elsif @meta[key].is_a? Hash
159
- @meta[key].sort.each {|k,v| puts "#{k}: #{v}"}
160
- else
161
- puts @meta[key]
162
- end
110
+ input = @tables[table]
111
+ Starscope::Matcher.new(value, input).query()
163
112
  end
164
113
 
165
- def dump_all
166
- @tables.keys.each {|tbl| dump_table(tbl)}
167
- end
114
+ def line_for_record(rec)
115
+ return rec[:line] if rec[:line]
168
116
 
169
- def summary
170
- ret = {}
117
+ file = @meta[:files][rec[:file]]
171
118
 
172
- @tables.each_key do |key|
173
- ret[key] = @tables[key].count
174
- end
119
+ return file[:lines][rec[:line_no]-1] if file[:lines]
120
+ end
175
121
 
176
- ret
122
+ def tables
123
+ @tables.keys
177
124
  end
178
125
 
179
- def query(table, value)
126
+ def records(table)
180
127
  raise NoTableError if not @tables[table]
181
- input = @tables[table]
182
- StarScope::Matcher.new(value, input).query()
183
- end
184
128
 
185
- def export_ctags(file)
186
- file.puts <<END
187
- !_TAG_FILE_FORMAT 2 /extended format/
188
- !_TAG_FILE_SORTED 1 /0=unsorted, 1=sorted, 2=foldcase/
189
- !_TAG_PROGRAM_AUTHOR Evan Huus /eapache@gmail.com/
190
- !_TAG_PROGRAM_NAME StarScope //
191
- !_TAG_PROGRAM_URL https://github.com/eapache/starscope //
192
- !_TAG_PROGRAM_VERSION #{StarScope::VERSION} //
193
- END
194
- defs = (@tables[:defs] || {}).sort_by {|x| x[:name][-1].to_s}
195
- defs.each do |record|
196
- file.puts StarScope::Record.ctag_line(record, @meta[:files][record[:file]])
197
- end
129
+ @tables[table]
198
130
  end
199
131
 
200
- # ftp://ftp.eeng.dcu.ie/pub/ee454/cygwin/usr/share/doc/mlcscope-14.1.8/html/cscope.html
201
- def export_cscope(file)
202
- buf = ""
203
- files = []
204
- db_by_line().each do |filename, lines|
205
- next if lines.empty?
206
-
207
- buf << "\t@#{filename}\n\n"
208
- buf << "0 #{CSCOPE_GLOBAL_HACK_START}\n"
209
- files << filename
210
- func_count = 0
211
-
212
- lines.sort.each do |line_no, records|
213
- line = records.first[:line]
214
- toks = tokenize_line(line, records)
215
- next if toks.empty?
216
-
217
- prev = 0
218
- buf << line_no.to_s << " "
219
- toks.each do |offset, record|
220
-
221
- next if offset < prev # this probably indicates an extractor bug
222
-
223
- # Don't export nested functions, cscope barfs on them since C doesn't
224
- # have them at all. Skipping tokens is easy; since prev isn't updated
225
- # they get turned into plain text automatically.
226
- if record[:type] == :func
227
- case record[:tbl]
228
- when :defs
229
- func_count += 1
230
- next unless func_count == 1
231
- when :end
232
- func_count -= 1
233
- next unless func_count == 0
234
- end
235
- end
236
-
237
- buf << CSCOPE_GLOBAL_HACK_STOP if record[:type] == :func && record[:tbl] == :defs
238
- buf << cscope_plaintext(line, prev, offset) << "\n"
239
- buf << StarScope::Record.cscope_mark(record[:tbl], record) << record[:key] << "\n"
240
- buf << CSCOPE_GLOBAL_HACK_START if record[:type] == :func && record[:tbl] == :end
241
-
242
- prev = offset + record[:key].length
132
+ def metadata(key=nil)
133
+ return @meta.keys if key.nil?
243
134
 
244
- end
245
- buf << cscope_plaintext(line, prev, line.length) << "\n\n"
246
- end
247
- end
248
-
249
- buf << "\t@\n"
250
-
251
- header = "cscope 15 #{Dir.pwd} -c "
252
- offset = "%010d\n" % (header.length + 11 + buf.bytes.count)
135
+ raise NoTableError unless @meta[key]
253
136
 
254
- file.print(header)
255
- file.print(offset)
256
- file.print(buf)
257
-
258
- file.print("#{@meta[:paths].length}\n")
259
- @meta[:paths].each {|p| file.print("#{p}\n")}
260
- file.print("0\n")
261
- file.print("#{files.length}\n")
262
- buf = ""
263
- files.each {|f| buf << f + "\n"}
264
- file.print("#{buf.length}\n#{buf}")
137
+ @meta[key]
265
138
  end
266
139
 
267
140
  private
268
141
 
269
- def add_new_files(files)
270
- files.each do |file|
271
- @output.log("Adding `#{file}`")
272
- parse_file(file)
273
- @output.inc_pbar
142
+ def open_db(filename)
143
+ File.open(filename, 'r') do |file|
144
+ begin
145
+ Zlib::GzipReader.wrap(file) do |stream|
146
+ parse_db(stream)
147
+ end
148
+ rescue Zlib::GzipFile::Error
149
+ file.rewind
150
+ parse_db(file)
151
+ end
274
152
  end
275
153
  end
276
154
 
277
- def update_files(files)
278
- remove_files(files)
279
- add_new_files(files)
155
+ # returns true iff the database is in the most recent format
156
+ def parse_db(stream)
157
+ case stream.gets.to_i
158
+ when DB_FORMAT
159
+ @meta = Oj.load(stream.gets)
160
+ @tables = Oj.load(stream.gets)
161
+ return true
162
+ when 3..4
163
+ # Old format, so read the directories segment then rebuild
164
+ add_paths(Oj.load(stream.gets))
165
+ return false
166
+ when 0..2
167
+ # Old format (pre-json), so read the directories segment then rebuild
168
+ len = stream.gets.to_i
169
+ add_paths(Marshal::load(stream.read(len)))
170
+ return false
171
+ else
172
+ raise UnknownDBFormatError
173
+ end
174
+ rescue Oj::ParseError
175
+ stream.rewind
176
+ raise unless stream.gets.to_i == DB_FORMAT
177
+ # try reading as formated json, which is much slower, but it is sometimes
178
+ # useful to be able to directly read your db
179
+ objects = []
180
+ Oj.load(stream) {|obj| objects << obj}
181
+ @meta, @tables = objects
182
+ return true
280
183
  end
281
184
 
282
- def remove_files(files)
283
- files.each do |file|
284
- @output.log("Removing `#{file}`")
285
- @meta[:files].delete(file)
286
- end
287
- files = files.to_set
288
- @tables.each do |name, tbl|
289
- tbl.delete_if {|val| files.include?(val[:file])}
290
- end
185
+ def fixup
186
+ # misc things that were't worth bumping the format for, but which might not be written by old versions
187
+ @meta[:langs] ||= {}
291
188
  end
292
189
 
293
190
  # File.fnmatch treats a "**" to match files and directories recursively
294
- def normalize_fnmatch(path)
191
+ def self.normalize_fnmatch(path)
295
192
  if path == "."
296
193
  "**"
297
194
  elsif File.directory?(path)
@@ -303,7 +200,7 @@ END
303
200
 
304
201
  # Dir.glob treats a "**" to only match directories recursively; you need
305
202
  # "**/*" to match all files recursively
306
- def normalize_glob(path)
203
+ def self.normalize_glob(path)
307
204
  if path == "."
308
205
  File.join("**", "*")
309
206
  elsif File.directory?(path)
@@ -313,93 +210,81 @@ END
313
210
  end
314
211
  end
315
212
 
316
- def db_by_line()
317
- db = {}
318
- @tables.each do |tbl, records|
319
- records.each do |record|
320
- next if not record[:line_no]
321
- record[:tbl] = tbl
322
- db[record[:file]] ||= {}
323
- db[record[:file]][record[:line_no]] ||= []
324
- db[record[:file]][record[:line_no]] << record
325
- end
326
- end
327
- return db
213
+ def matches_exclude?(file, patterns = @meta[:excludes])
214
+ patterns.map {|p| File.fnmatch(p, file)}.any?
328
215
  end
329
216
 
330
- def tokenize_line(line, records)
331
- toks = {}
332
-
333
- records.each do |record|
334
- key = record[:name][-1].to_s
335
-
336
- # use the column if we have it, otherwise fall back to scanning
337
- index = record[:col] || line.index(key)
338
-
339
- # keep scanning if our current index doesn't actually match the key, or if
340
- # either the preceeding or succeeding character is a word character
341
- # (meaning we've accidentally matched the middle of some other token)
342
- while !index.nil? &&
343
- ((line[index, key.length] != key) ||
344
- (index > 0 && line[index-1] =~ /\w/) ||
345
- (index+key.length < line.length && line[index+key.length] =~ /\w/))
346
- index = line.index(key, index+1)
347
- end
348
-
349
- next if index.nil?
350
-
351
- # Strip trailing non-word characters, otherwise cscope barfs on
352
- # function names like `include?`
353
- if key =~ /^\W*$/
354
- next unless [:defs, :end].include?(record[:tbl])
355
- else
356
- key.sub!(/\W+$/, '')
357
- end
358
-
359
- record[:key] = key
360
- toks[index] = record
361
-
217
+ def add_files(files)
218
+ files.each do |file|
219
+ @output.extra("Adding `#{file}`")
220
+ parse_file(file)
221
+ @output.inc_pbar
362
222
  end
363
-
364
- return toks.sort
365
223
  end
366
224
 
367
- def cscope_plaintext(line, start, stop)
368
- ret = line.slice(start, stop-start)
369
- ret.lstrip! if start == 0
370
- ret.rstrip! if stop == line.length
371
- ret.gsub(/\s+/, ' ')
372
- rescue ArgumentError
373
- # invalid utf-8 byte sequence in the line, oh well
374
- line
225
+ def remove_files(files)
226
+ files.each do |file|
227
+ @output.extra("Removing `#{file}`")
228
+ @meta[:files].delete(file)
229
+ end
230
+ files = files.to_set
231
+ @tables.each do |name, tbl|
232
+ tbl.delete_if {|val| files.include?(val[:file])}
233
+ end
375
234
  end
376
235
 
377
- def matches_exclude?(patterns, file)
378
- patterns.map {|p| File.fnmatch(p, file)}.any?
236
+ def update_files(files)
237
+ remove_files(files)
238
+ add_files(files)
379
239
  end
380
240
 
381
241
  def parse_file(file)
382
242
  @meta[:files][file] = {:last_updated => File.mtime(file).to_i}
383
243
 
384
- LANGS.each do |lang|
385
- next if not lang.match_file file
386
- lang.extract file do |tbl, name, args|
244
+ EXTRACTORS.each do |extractor|
245
+ next if not extractor.match_file file
246
+
247
+ lines = nil
248
+ line_cache = nil
249
+ extractor.extract file do |tbl, name, args|
387
250
  @tables[tbl] ||= []
388
- @tables[tbl] << StarScope::Record.build(file, name, args)
251
+ @tables[tbl] << self.class.normalize_record(file, name, args)
252
+
253
+ if args[:line_no]
254
+ line_cache ||= File.readlines(file)
255
+ lines ||= Array.new(line_cache.length)
256
+ lines[args[:line_no]-1] = line_cache[args[:line_no]-1].chomp
257
+ end
389
258
  end
390
- @meta[:files][file][:lang] = lang.name.split('::').last.to_sym
259
+
260
+ @meta[:files][file][:lang] = extractor.name.split('::').last.to_sym
261
+ @meta[:files][file][:lines] = lines
391
262
  return
392
263
  end
393
264
  end
394
265
 
395
266
  def file_changed(name)
396
- if not File.exists?(name) or not File.file?(name)
267
+ file_meta = @meta[:files][name]
268
+ if !File.exists?(name) || !File.file?(name)
397
269
  :deleted
398
- elsif @meta[:files][name][:last_updated] < File.mtime(name).to_i
270
+ elsif (file_meta[:last_updated] < File.mtime(name).to_i) ||
271
+ (file_meta[:lang] && (@meta[:langs][file_meta[:lang]] || 0) < LANGS[file_meta[:lang]])
399
272
  :modified
400
273
  else
401
274
  :unchanged
402
275
  end
403
276
  end
404
277
 
278
+ def self.normalize_record(file, name, args)
279
+ args[:file] = file
280
+
281
+ if name.is_a? Array
282
+ args[:name] = name.map {|x| x.to_sym}
283
+ else
284
+ args[:name] = [name.to_sym]
285
+ end
286
+
287
+ args
288
+ end
289
+
405
290
  end