ghtorrent 0.4 → 0.5

Sign up to get free protection for your applications and to get access to all the features.
Files changed (38) hide show
  1. data/CHANGELOG +24 -0
  2. data/Gemfile +17 -0
  3. data/Gemfile.lock +40 -0
  4. data/README.md +23 -22
  5. data/bin/ght-data-retrieval +66 -24
  6. data/bin/ght-load +41 -19
  7. data/bin/ght-mirror-events +13 -16
  8. data/bin/ght-rm-dupl +119 -77
  9. data/lib/ghtorrent.rb +14 -4
  10. data/lib/ghtorrent/adapters/base_adapter.rb +17 -5
  11. data/lib/ghtorrent/adapters/mongo_persister.rb +122 -56
  12. data/lib/ghtorrent/api_client.rb +151 -16
  13. data/lib/ghtorrent/bson_orderedhash.rb +23 -0
  14. data/lib/ghtorrent/cache.rb +97 -0
  15. data/lib/ghtorrent/command.rb +43 -25
  16. data/lib/ghtorrent/gh_torrent_exception.rb +6 -0
  17. data/lib/ghtorrent/ghtorrent.rb +615 -164
  18. data/lib/ghtorrent/hash.rb +11 -0
  19. data/lib/ghtorrent/logging.rb +11 -7
  20. data/lib/ghtorrent/migrations/001_init_schema.rb +3 -3
  21. data/lib/ghtorrent/migrations/002_add_external_ref_ids.rb +2 -0
  22. data/lib/ghtorrent/migrations/003_add_orgs.rb +4 -1
  23. data/lib/ghtorrent/migrations/004_add_commit_comments.rb +4 -2
  24. data/lib/ghtorrent/migrations/005_add_repo_collaborators.rb +2 -0
  25. data/lib/ghtorrent/migrations/006_add_watchers.rb +2 -0
  26. data/lib/ghtorrent/migrations/007_add_pull_requests.rb +64 -0
  27. data/lib/ghtorrent/migrations/008_add_project_unq.rb +23 -0
  28. data/lib/ghtorrent/migrations/009_add_project_commit.rb +27 -0
  29. data/lib/ghtorrent/migrations/010_add_forks.rb +28 -0
  30. data/lib/ghtorrent/migrations/mysql_defaults.rb +6 -0
  31. data/lib/ghtorrent/persister.rb +3 -0
  32. data/lib/ghtorrent/retriever.rb +298 -102
  33. data/lib/ghtorrent/settings.rb +20 -1
  34. data/lib/ghtorrent/time.rb +5 -0
  35. data/lib/ghtorrent/utils.rb +22 -4
  36. data/lib/version.rb +5 -0
  37. metadata +173 -145
  38. data/lib/ghtorrent/call_stack.rb +0 -91
data/lib/ghtorrent/call_stack.rb DELETED
@@ -1,91 +0,0 @@
1
- module GHTorrent
2
- class CallStack
3
-
4
- @@callstacks = Hash.new
5
-
6
- attr_reader :name
7
-
8
- def self.new(*args)
9
- name = args[0]
10
- if @@callstacks.has_key? name
11
- @@callstacks[name]
12
- else
13
- o = allocate
14
- if o.__send__(:initialize, *args)
15
- @@callstacks[name] = o
16
- o
17
- else
18
- nil
19
- end
20
- end
21
- end
22
-
23
- def initialize(name, sync_every = 5)
24
-
25
- @stack = Array.new
26
- @name = name
27
- @sync = sync_every
28
-
29
- if File.exists?(name)
30
- @file = File.new(name, "r")
31
- puts "File #{name} exists, importing stack..."
32
- read = @file.readlines.reverse.reduce(0) { |acc, x|
33
- @stack.push x
34
- acc
35
- }
36
- puts "\n#{read} entries read"
37
- @file.close
38
- end
39
-
40
- flusher = Thread.new {
41
- while true
42
- begin
43
- if not @stack.empty?
44
- @file = File.new(name, "w+")
45
- @stack.each { |l| @file.write("#{l} \n") }
46
- @file.fsync
47
- @file.close
48
- end
49
- sleep(@sync)
50
- rescue
51
- puts "flusher thread failed for #{name}"
52
- end
53
- end
54
- }
55
-
56
- ObjectSpace.define_finalizer(self, proc {
57
- puts "Finalizer: Cleaning up #{@name}"
58
- @@callstacks.delete[@name]
59
- flusher.stop
60
- cleanup
61
- })
62
-
63
- at_exit { cleanup }
64
- end
65
-
66
- def push(item)
67
- @stack.push(item)
68
- end
69
-
70
- def pop()
71
- @stack.pop
72
- end
73
-
74
- def empty
75
- @stack.delete_if { |x| true }
76
- end
77
-
78
- private
79
-
80
- def cleanup
81
- if @stack.empty?
82
- if File.exists? @name
83
- puts "removing stack #{@name}"
84
- File.delete(@name)
85
- end
86
- else
87
- puts "stack #{@name} contains #{@stack.size} items"
88
- end
89
- end
90
- end
91
- end