mesh-rb 0.0.1 → 0.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (124) hide show
  1. checksums.yaml +4 -4
  2. data/Gemfile.lock +1 -1
  3. data/ext/mesh/extconf.rb +22 -4
  4. data/ext/mesh/mesh.tar.gz +0 -0
  5. data/lib/mesh/version.rb +1 -1
  6. data/mesh.gemspec +3 -2
  7. metadata +4 -120
  8. data/ext/mesh/mesh/.bazelrc +0 -20
  9. data/ext/mesh/mesh/.bazelversion +0 -1
  10. data/ext/mesh/mesh/.clang-format +0 -15
  11. data/ext/mesh/mesh/.dockerignore +0 -5
  12. data/ext/mesh/mesh/.editorconfig +0 -16
  13. data/ext/mesh/mesh/.gitattributes +0 -4
  14. data/ext/mesh/mesh/.github/workflows/main.yml +0 -144
  15. data/ext/mesh/mesh/.gitignore +0 -51
  16. data/ext/mesh/mesh/AUTHORS +0 -5
  17. data/ext/mesh/mesh/CMakeLists.txt +0 -270
  18. data/ext/mesh/mesh/CODE_OF_CONDUCT.md +0 -77
  19. data/ext/mesh/mesh/Dockerfile +0 -30
  20. data/ext/mesh/mesh/LICENSE +0 -201
  21. data/ext/mesh/mesh/Makefile +0 -81
  22. data/ext/mesh/mesh/README.md +0 -97
  23. data/ext/mesh/mesh/WORKSPACE +0 -50
  24. data/ext/mesh/mesh/bazel +0 -350
  25. data/ext/mesh/mesh/mesh-pldi19-powers.pdf +0 -0
  26. data/ext/mesh/mesh/src/BUILD +0 -222
  27. data/ext/mesh/mesh/src/CMakeLists.txt +0 -85
  28. data/ext/mesh/mesh/src/bitmap.h +0 -590
  29. data/ext/mesh/mesh/src/cheap_heap.h +0 -170
  30. data/ext/mesh/mesh/src/common.h +0 -377
  31. data/ext/mesh/mesh/src/copts.bzl +0 -31
  32. data/ext/mesh/mesh/src/d_assert.cc +0 -75
  33. data/ext/mesh/mesh/src/fixed_array.h +0 -124
  34. data/ext/mesh/mesh/src/global_heap.cc +0 -547
  35. data/ext/mesh/mesh/src/global_heap.h +0 -569
  36. data/ext/mesh/mesh/src/gnu_wrapper.cc +0 -75
  37. data/ext/mesh/mesh/src/internal.h +0 -356
  38. data/ext/mesh/mesh/src/libmesh.cc +0 -239
  39. data/ext/mesh/mesh/src/mac_wrapper.cc +0 -528
  40. data/ext/mesh/mesh/src/measure_rss.cc +0 -44
  41. data/ext/mesh/mesh/src/measure_rss.h +0 -20
  42. data/ext/mesh/mesh/src/meshable_arena.cc +0 -776
  43. data/ext/mesh/mesh/src/meshable_arena.h +0 -309
  44. data/ext/mesh/mesh/src/meshing.h +0 -60
  45. data/ext/mesh/mesh/src/mini_heap.h +0 -532
  46. data/ext/mesh/mesh/src/mmap_heap.h +0 -104
  47. data/ext/mesh/mesh/src/one_way_mmap_heap.h +0 -77
  48. data/ext/mesh/mesh/src/partitioned_heap.h +0 -111
  49. data/ext/mesh/mesh/src/plasma/mesh.h +0 -33
  50. data/ext/mesh/mesh/src/real.cc +0 -52
  51. data/ext/mesh/mesh/src/real.h +0 -36
  52. data/ext/mesh/mesh/src/rng/mwc.h +0 -296
  53. data/ext/mesh/mesh/src/rng/mwc64.h +0 -58
  54. data/ext/mesh/mesh/src/rpl_printf.c +0 -1991
  55. data/ext/mesh/mesh/src/runtime.cc +0 -393
  56. data/ext/mesh/mesh/src/runtime.h +0 -114
  57. data/ext/mesh/mesh/src/shuffle_vector.h +0 -287
  58. data/ext/mesh/mesh/src/size_classes.def +0 -251
  59. data/ext/mesh/mesh/src/static/if.h +0 -36
  60. data/ext/mesh/mesh/src/static/log.h +0 -43
  61. data/ext/mesh/mesh/src/testing/benchmark/local_refill.cc +0 -103
  62. data/ext/mesh/mesh/src/testing/big-alloc.c +0 -28
  63. data/ext/mesh/mesh/src/testing/fragmenter.cc +0 -128
  64. data/ext/mesh/mesh/src/testing/global-large-stress.cc +0 -25
  65. data/ext/mesh/mesh/src/testing/local-alloc.c +0 -16
  66. data/ext/mesh/mesh/src/testing/meshing_benchmark.cc +0 -189
  67. data/ext/mesh/mesh/src/testing/thread.cc +0 -35
  68. data/ext/mesh/mesh/src/testing/unit/alignment.cc +0 -56
  69. data/ext/mesh/mesh/src/testing/unit/bitmap_test.cc +0 -274
  70. data/ext/mesh/mesh/src/testing/unit/concurrent_mesh_test.cc +0 -185
  71. data/ext/mesh/mesh/src/testing/unit/mesh_test.cc +0 -143
  72. data/ext/mesh/mesh/src/testing/unit/rng_test.cc +0 -22
  73. data/ext/mesh/mesh/src/testing/unit/size_class_test.cc +0 -66
  74. data/ext/mesh/mesh/src/testing/unit/triple_mesh_test.cc +0 -285
  75. data/ext/mesh/mesh/src/testing/userfaultfd-kernel-copy.cc +0 -164
  76. data/ext/mesh/mesh/src/thread_local_heap.cc +0 -163
  77. data/ext/mesh/mesh/src/thread_local_heap.h +0 -268
  78. data/ext/mesh/mesh/src/wrapper.cc +0 -433
  79. data/ext/mesh/mesh/support/export_mesh.cmake +0 -28
  80. data/ext/mesh/mesh/support/gen-size-classes +0 -57
  81. data/ext/mesh/mesh/support/install_all_configs +0 -33
  82. data/ext/mesh/mesh/support/remove_export_mesh.cmake +0 -48
  83. data/ext/mesh/mesh/support/update-bazelisk +0 -8
  84. data/ext/mesh/mesh/theory/32m80.png +0 -0
  85. data/ext/mesh/mesh/theory/64m80ind.png +0 -0
  86. data/ext/mesh/mesh/theory/bound_comparison.py +0 -67
  87. data/ext/mesh/mesh/theory/bounds/impdeg+1 +0 -135
  88. data/ext/mesh/mesh/theory/choose.py +0 -43
  89. data/ext/mesh/mesh/theory/common.py +0 -42
  90. data/ext/mesh/mesh/theory/compute_exp_Y.py +0 -134
  91. data/ext/mesh/mesh/theory/createRandomString.py +0 -69
  92. data/ext/mesh/mesh/theory/deg_bound_check.py +0 -100
  93. data/ext/mesh/mesh/theory/degcheck.py +0 -47
  94. data/ext/mesh/mesh/theory/dumps/32,1,80,dumb.txt +0 -81
  95. data/ext/mesh/mesh/theory/dumps/32,2,80,dumb.txt +0 -81
  96. data/ext/mesh/mesh/theory/dumps/32,3,80,dumb.txt +0 -81
  97. data/ext/mesh/mesh/theory/dumps/32,4,80,dumb.txt +0 -81
  98. data/ext/mesh/mesh/theory/dumps/32,5,80,dumb.txt +0 -81
  99. data/ext/mesh/mesh/theory/dumps/32,6,80,dumb.txt +0 -81
  100. data/ext/mesh/mesh/theory/dumps/32,7,80,dumb.txt +0 -81
  101. data/ext/mesh/mesh/theory/dumps/32,8,80,dumb.txt +0 -81
  102. data/ext/mesh/mesh/theory/dumps/32,9,80,dumb.txt +0 -81
  103. data/ext/mesh/mesh/theory/experiment.py +0 -303
  104. data/ext/mesh/mesh/theory/experiment_raw_results/.gitignore +0 -0
  105. data/ext/mesh/mesh/theory/greedy_experiment.py +0 -66
  106. data/ext/mesh/mesh/theory/greedy_experiment_copy.py +0 -46
  107. data/ext/mesh/mesh/theory/greedy_experiment_q.py +0 -75
  108. data/ext/mesh/mesh/theory/makeGraph.py +0 -64
  109. data/ext/mesh/mesh/theory/manyreps.png +0 -0
  110. data/ext/mesh/mesh/theory/manystrings.png +0 -0
  111. data/ext/mesh/mesh/theory/match_vs_color_experiment.py +0 -94
  112. data/ext/mesh/mesh/theory/maxmatch_vs_E[Y].py +0 -162
  113. data/ext/mesh/mesh/theory/maxmatch_vs_greedymatch.py +0 -96
  114. data/ext/mesh/mesh/theory/maxvdeg+1imp++32,80.png +0 -0
  115. data/ext/mesh/mesh/theory/mesh_util.py +0 -322
  116. data/ext/mesh/mesh/theory/meshers.py +0 -452
  117. data/ext/mesh/mesh/theory/meshingBenchmark.py +0 -96
  118. data/ext/mesh/mesh/theory/occupancyComparison.py +0 -133
  119. data/ext/mesh/mesh/theory/randmatch_vs_greedymatch.py +0 -97
  120. data/ext/mesh/mesh/theory/randmatch_vs_greedymatch_q.py +0 -103
  121. data/ext/mesh/mesh/theory/randmatch_vs_greedymatch_time.py +0 -117
  122. data/ext/mesh/mesh/theory/read_mesh_dump.py +0 -82
  123. data/ext/mesh/mesh/theory/test.py +0 -70
  124. data/ext/mesh/mesh/tools/bazel +0 -1
@@ -1,96 +0,0 @@
1
- # -*- coding: utf-8 -*-
2
- """
3
- Created on Fri Apr 15 15:25:44 2016
4
-
5
- @author: devd
6
- """
7
-
8
- from __future__ import division
9
- from createRandomString import *
10
- from makeGraph import *
11
- from greedyMesher import *
12
- import matplotlib.pyplot as plt
13
- import matplotlib.patches as mpatches
14
- import networkx as nx
15
- import numpy as np
16
- import time
17
-
18
- def experiment(length, ones_range_min, ones_range_max, reps, numStrings):
19
- strings = []
20
- ones = []
21
- maxmatch_avg = []
22
- maxmatch_std_dev = []
23
- greedymatch_avg = []
24
- greedymatch_std_dev = []
25
-
26
- for numOnes in range(ones_range_min, ones_range_max+1):
27
- ones.append(numOnes)
28
- freed_pages_maxmatching = []
29
- freed_pages_greedymatching = []
30
- for iterations in range (reps):
31
- for i in range(numStrings):
32
- strings.append(createRandomString(length, numOnes))
33
-
34
- graph = makeGraph(strings)
35
- frdpgs_maxmatching = len(nx.max_weight_matching(graph))/2
36
- perc = (frdpgs_maxmatching/numStrings)*100
37
- freed_pages_maxmatching.append(perc)
38
-
39
- # graph_c = nx.complement(graph)
40
- # frdpgs_greedymatching = numStrings - color_counter(graph_c)
41
- # perc = (frdpgs_greedymatching/numStrings)*100
42
- # freed_pages_greedymatching.append(perc)
43
-
44
- b, unmatched = greedyMesher(strings)
45
- frdpgs_greedymatching = (numStrings - len(unmatched))/2
46
- perc = (frdpgs_greedymatching/numStrings)*100
47
- freed_pages_greedymatching.append(perc)
48
-
49
- strings = []
50
- m = np.asarray(freed_pages_maxmatching)
51
- m_a = np.mean(m)
52
- maxmatch_avg.append(m_a)
53
- m_s = np.std(m)
54
- maxmatch_std_dev.append(m_s)
55
-
56
- c = np.asarray(freed_pages_greedymatching)
57
- c_a = np.mean(c)
58
- greedymatch_avg.append(c_a)
59
- c_s = np.std(c)
60
- greedymatch_std_dev.append(c_s)
61
-
62
- return ones, maxmatch_avg, maxmatch_std_dev, greedymatch_avg, greedymatch_std_dev
63
-
64
-
65
- def plot_it(length, ones_range_min, ones_range_max, reps, numStrings):
66
- ones, match_avg, match_std_dev, color_avg, color_std_dev = experiment(length, ones_range_min, ones_range_max, reps, numStrings)
67
-
68
- plt.errorbar(np.asarray(ones), np.asarray(match_avg), np.asarray(match_std_dev), markersize=3, lw=1, fmt='-o')
69
- plt.errorbar(np.asarray(ones), np.asarray(color_avg), np.asarray(color_std_dev), markersize=3, lw=1, fmt='-o')
70
- plt.ylim([0,60])
71
- plt.ylabel('Percentage of pages freed')
72
- plt.xlabel('Number of objects per page')
73
- blue_patch = mpatches.Patch(color='blue', label='max matching')
74
- green_patch = mpatches.Patch(color = 'green', label = 'greedy matching')
75
- plt.legend(handles=[blue_patch, green_patch])
76
- plt.title('MAX MATCHING VS GREEDY MATCHING MESHING RESULTS \n{}-object pages, {} pages'.format(length, numStrings))
77
- #plt.show()
78
- plt.savefig('maxvgreedy{},{}'.format(length, numStrings) + '.png', dpi = 1000)
79
- plt.close()
80
-
81
- #length = [32,64]
82
- length = [128]
83
- ones_range_min = 1
84
- ones_range_max = 32
85
- reps = 10
86
- #numStrings = [80,100,150,200]
87
- numStrings= [150,200]
88
-
89
-
90
- start = time.time()
91
- for l in length:
92
- for n in numStrings:
93
- plot_it(l, ones_range_min, int(l/2), reps, n)
94
- print 'max match vs greedy match plot {},{} done'.format(l,n)
95
- end = time.time()
96
- print('making this took {} seconds'.format(end-start) )
@@ -1,322 +0,0 @@
1
- # -*- coding: utf-8 -*-
2
- """
3
- Created on Mon Jun 19 15:32:02 2017
4
-
5
- @author: devd
6
- """
7
- import math
8
- import operator
9
- from itertools import izip, imap
10
- import random
11
- from scipy.misc import comb as fast_nCr
12
- from scipy.special import gamma
13
-
14
-
15
- def formatStrings(strings):
16
- """Adds extra data to a list of strings for ease of meshing. Replaces each
17
- string in the list with a tuple (A,B,C,D). A = original string. B = binary
18
- representation for fast arithmetic. C = occupancy. D = flag that indicates
19
- whether the string has been meshed(initially set to False)."""
20
- new_strings = []
21
- for string in strings:
22
- #new_strings.append((string, long(string, base=2)))
23
- new_strings.append(
24
- (string, long(string, base=2), string.count("1"), False))
25
- return new_strings
26
-
27
-
28
- def hamming(str1, str2):
29
- """Calculates the Hamming distance between two strings of equal length."""
30
- # if type(str1) == long:
31
- # str1 = bin(str1)[2:].rjust(len(str2),"0")
32
- assert len(str1) == len(str2)
33
- ne = operator.ne
34
- return sum(imap(ne, str1, str2))
35
-
36
-
37
- def fast_q(length, occ1, occ2):
38
- """computes the probability that two strings with given occupancies will
39
- mesh."""
40
- result = float((fast_nCr(length - occ2, occ1))) / (fast_nCr(length, occ1))
41
- return result
42
-
43
-
44
- def faster_q(length, occ1, occ2):
45
- numerator = 1
46
- for i in range(length - occ1, length - occ1 - occ2, -1):
47
- print(i)
48
- numerator *= i
49
- denominator = 1
50
- for i in range(length, length - occ2, -1):
51
- denominator *= i
52
- return float(numerator) / float(denominator)
53
-
54
-
55
- def generate_cutoffs(bkt1, length, cutoff):
56
- """returns a dict indexed by string occupancy, value is the cutoff occupancy
57
- for potential meshes (if you encounter a higher occupancy during a greedy
58
- search for a mesh, stop)."""
59
- cutoffs = {}
60
- for s in bkt1:
61
- occ1 = s[2]
62
- if occ1 not in cutoffs.keys():
63
- cutoffs[occ1] = float('inf')
64
- # only calculate cutoffs for every 5th occupancy, to save time
65
- for occ2 in range(0, int(length / 2), 5):
66
- if faster_q(length, occ1, occ2) < cutoff:
67
- cutoffs[occ1] = occ2
68
- break
69
- return cutoffs
70
-
71
-
72
- class Splitter(object):
73
- """
74
- Encapsulates splitting behavior for a trial.
75
-
76
- Keeps track of multiple different splitting strings and can
77
- automatically cycle through them if required.
78
- """
79
-
80
- def __init__(self, length):
81
- self.length = length
82
- self.splitting_strings = []
83
- self.num_splitters = int(math.log(length, 2))
84
- # print self.num_splitters
85
- for i in range(1, self.num_splitters + 1):
86
- split_string = ""
87
- for j in range(2**(i - 1)):
88
- split_string = split_string + \
89
- (("1" * int((length / (2**i)))) +
90
- ("0" * (int(length / (2**i)))))
91
- self.splitting_strings.append(split_string)
92
- # print self.splitting_strings
93
- print 'Splitter(%d): %d splitters with strings: %s' % \
94
- (length, self.num_splitters, self.splitting_strings)
95
-
96
- self.current_method = 0
97
-
98
- def _splitter(self, strings, advance):
99
- """splits the given string set based on the current splitting string.
100
- optionally advances to the next splitting string for future splittings."""
101
- split = self.splitting_strings[self.current_method]
102
- if advance:
103
- self.current_method = self.current_method + 1
104
- bucket1 = []
105
- bucket2 = []
106
- for s in strings:
107
- diff = hamming(s[0], split)
108
- if diff < int(self.length * 0.5):
109
- bucket1.append(s)
110
- elif diff == int(self.length * 0.5):
111
- if random.randint(0, 1):
112
- bucket1.append(s)
113
- else:
114
- bucket2.append(s)
115
- else:
116
- bucket2.append(s)
117
- return bucket1, bucket2
118
-
119
- def split(self, strings=[], bucket1=[], bucket2=[], advance=True):
120
- """the outward-facing method for splitting. gracefully handles both
121
- a single string set and a """
122
- # print 'trying to split. current method is {}'.format(self.current_method)
123
- if strings == [] and bucket1 == [] and bucket2 == []:
124
- raise Exception('must provide split method with nonempty input')
125
- if strings != []:
126
- return self._splitter(strings, advance)
127
- else:
128
- if self.current_method >= self.num_splitters:
129
- return bucket1, bucket2
130
- else:
131
- return self._splitter(bucket1 + bucket2, advance)
132
-
133
- def advance(self):
134
- self.current_method = self.current_method + 1
135
-
136
-
137
- def occupancySort(strings):
138
- """Modifies given list of strings in place, sorting them in order of
139
- increasing occupancy."""
140
- # strings.sort(key = lambda x: x[0].count("1"))
141
- strings.sort(key=lambda x: x[2])
142
-
143
-
144
- def simple_traverse(meshes, strings, dim=0):
145
- """probes a list of strings for meshable pairs. the first string is checked
146
- against the second, third/fourth, etc. mesh and unmeshed string lists are
147
- modified in place. returns True if all strings have been meshed; else returns
148
- False."""
149
- # print 'here are the strings passed to simple_traverse', strings
150
- # print 'and dim is', dim
151
- matched = []
152
- for i in range(len(strings) - 2, -1 + dim, -2):
153
- num1 = strings[i][1]
154
- num2 = strings[i + 1][1]
155
- # print num1, num2
156
- if num1 & num2 == 0:
157
- matched.append(i)
158
- meshes.append((strings[i], strings[i + 1]))
159
- # meshes.append(strings[i+1])
160
- # print "adding mesh {}, {}".format(strings[i], strings[i+1])
161
- for x in matched:
162
- del strings[x + 1]
163
- del strings[x]
164
- if len(strings) == 0:
165
- return True
166
- return False
167
-
168
-
169
- def traverse(meshes, bucket1=None, bucket2=None, strings=None, extra=False):
170
- """looks for meshable pairs between the buckets. modifies the buckets and
171
- the list of found meshes in place. returns whether or not meshing is done.
172
- throws an assertion error if only one bucket has anything in it, so the
173
- caller can resplit the buckets or whatever."""
174
- if strings != None:
175
- # print 'found strings'
176
- return simple_traverse(strings, meshes)
177
- if bucket1 == None or bucket2 == None:
178
- raise Exception(
179
- 'must pass either buckets or string set to traverse function')
180
-
181
- dim = min(len(bucket1), len(bucket2))
182
- if len(bucket1) == len(bucket2) == 0:
183
- return True
184
- assert dim != 0
185
- matched = []
186
- if dim == 1:
187
- num1 = bucket1[0][1]
188
- num2 = bucket2[0][1]
189
- if num1 & num2 == 0:
190
- matched.append(0)
191
- for i in range(dim - 1, 0, -1):
192
- num1 = bucket1[i][1]
193
- num2 = bucket2[i][1]
194
- if num1 & num2 == 0:
195
- matched.append(i)
196
- for x in matched:
197
- meshes.append((bucket1[x], bucket2[x]))
198
- # if one bucket is larger than the other, mesh remaining strings among themselves
199
- if extra:
200
- # print 'extra'
201
- if len(bucket1) != len(bucket2):
202
- # print bucket1, bucket2
203
- # print 'chosing one'
204
- bucket = max([bucket1, bucket2], key=lambda x: len(x))
205
- # print '{} chosen'.format(bucket)
206
- simple_traverse(meshes, bucket, dim)
207
- # print bucket
208
-
209
- for x in matched:
210
- del bucket1[x]
211
- del bucket2[x]
212
- return False
213
-
214
-
215
- def simpleGreedyTraverse(meshes, strings, cutoff=None):
216
- """given a list of strings, exhaustively checks the first string for meshes,
217
- then the second, etc. found meshes are removed from the list. ends when all
218
- pairs of remaining strings have been checked. returns whether or not all
219
- strings have been meshed."""
220
-
221
- length = len(strings)
222
- strlength = len(strings[0][0])
223
- # matched = []
224
- if cutoff:
225
- cutoffs = generate_cutoffs(strings, strlength, cutoff)
226
- for i in range(length):
227
- # if the current string has already been meshed, skip it
228
- if strings[i][3]:
229
- continue
230
-
231
- if cutoff:
232
- current_cutoff = cutoffs[strings[i][2]]
233
- for j in range(i + 1, length):
234
- # if current string has already been meshed, skip it
235
- if strings[j][3]:
236
- continue
237
-
238
- if cutoff and strings[j][2] >= current_cutoff:
239
- break
240
-
241
- # if i not in matched and j not in matched: (should be unnecessary now, test soon)
242
- if not strings[i][3] and not strings[j][3]:
243
- num1 = strings[i][1]
244
- num2 = strings[j][1]
245
- if num1 & num2 == 0:
246
- # matched.append(i)
247
- # matched.append(j)
248
- strings[i] = (strings[i][0], strings[i]
249
- [1], strings[i][2], True)
250
- strings[j] = (strings[j][0], strings[j]
251
- [1], strings[j][2], True)
252
- meshes.append((strings[i], strings[j]))
253
- break
254
- for string1, string2 in meshes:
255
- strings.remove(string1)
256
- strings.remove(string2)
257
- if len(strings) == 0:
258
- return True
259
- return False
260
-
261
-
262
- def greedyTraverse(meshes, bucket1=None, bucket2=None, strings=None, cutoff=None):
263
- """
264
- Looks for meshable pairs between the buckets greedily (looks
265
- first at all potential meshes with the first string in bucket1 and
266
- anything in bucket 2, then the second string in bucket 2 with
267
- everything in bucket 2, etc. adds found pairs to meshes in
268
- place. returns whether or not all strings have been meshed.
269
- """
270
-
271
- # if only one string list is supplied, search it exhaustively for
272
- # pairs using a simpler function
273
- if strings != None:
274
- return simpleGreedyTraverse(meshes, strings, cutoff)
275
-
276
- if bucket1 == None or bucket2 == None:
277
- raise Exception(
278
- 'must pass either buckets or string set to traverse function')
279
-
280
- strlength = len(bucket1[0][0])
281
- len1, len2 = len(bucket1), len(bucket2)
282
- assert len1 != 0 and len2 != 0
283
- if cutoff:
284
- cutoffs = generate_cutoffs(bucket1, strlength, cutoff)
285
- for i in range(len1):
286
- if cutoff:
287
- bkt1cutoff = cutoffs[bucket1[i][2]]
288
-
289
- for j in range(len2):
290
- # notice when (due to occupancy ordering) there is little hope of finding more meshes
291
- # for the ith string in bucket 1
292
- if cutoff and bucket2[j][2] >= bkt1cutoff:
293
- # print "doing a break!"
294
- break
295
- if not bucket1[i][3] and not bucket2[j][3]:
296
- num1 = bucket1[i][1]
297
- num2 = bucket2[j][1]
298
- if num1 & num2 == 0:
299
- bucket1[i] = (bucket1[i][0], bucket1[i]
300
- [1], bucket1[i][2], True)
301
- bucket2[j] = (bucket2[j][0], bucket2[j]
302
- [1], bucket2[j][2], True)
303
- meshes.append((bucket1[i], bucket2[j]))
304
- for string1, string2 in meshes:
305
- # print "removing {} from bucket1 and {} from bucket2".format(string1, string2)
306
- bucket1.remove(string1)
307
- bucket2.remove(string2)
308
- if len(bucket1) == len(bucket2) == 0:
309
- return True
310
- return False
311
-
312
-
313
- if __name__ == '__main__':
314
- bkt1 = formatStrings([("11100000"), ("11111000")])
315
- bkt2 = formatStrings([("00011111"), ("00000111")])
316
- meshes = []
317
- greedyTraverse(meshes, bucket1=bkt1, bucket2=bkt2, cutoff=None)
318
- # occupancySort(bkt1)
319
- print bkt1, bkt2, meshes
320
- # print fast_q(64, 25,13)
321
- # print generate_cutoffs(bkt1, 8)
322
- # print generate_cutoffs(bkt2, 8)
@@ -1,452 +0,0 @@
1
- # -*- coding: utf-8 -*-
2
- """
3
- Created on Fri Feb 03 10:23:57 2017
4
-
5
- @author: devd
6
- """
7
-
8
- import random
9
- import numpy as np
10
- from makeGraph import makeGraph
11
- import networkx as nx
12
- from mesh_util import Splitter, hamming, traverse, occupancySort, formatStrings, simpleGreedyTraverse, greedyTraverse
13
- import time
14
-
15
-
16
- def simpleMesher(strings, stdOutput=False):
17
- """
18
- Attempts to mesh the first string in the list with the second, etc.
19
- Returns number of successful meshes.
20
- """
21
- meshes = 0
22
- # try to mesh each string
23
- for i in range(0, len(strings), 2):
24
- str1 = strings[i]
25
- str2 = strings[i + 1]
26
- num = [int(x) for x in list(str1)]
27
- num2 = [int(x) for x in list(str2)]
28
- if np.dot(num, num2) == 0:
29
- meshes += 1
30
- if stdOutput:
31
- return 100 * float(meshes) / len(strings)
32
- return meshes
33
-
34
-
35
- # def randomMesher(strings, attempts):
36
- # """DEPRECATED"""
37
- # s = [x for x in strings]
38
- # matched_strings = []
39
- # for i in range(attempts):
40
- # pair = random.sample(s,2)
41
- # str1 = pair[0]
42
- # str2 = pair[1]
43
- # num = [int(x) for x in list(str1)]
44
- # num2 = [int(x) for x in list(str2)]
45
- # if np.dot(num, num2) == 0:
46
- # #print('removing {} and {}'.format(str1, str2))
47
- # matched_strings.append(str1)
48
- # matched_strings.append(str2)
49
- # s.remove(str1)
50
- # s.remove(str2)
51
- # if len(s) < 2:
52
- # return matched_strings
53
- # return matched_strings
54
-
55
- def randomMesher(strings, attempts, display=False, stdOutput=False):
56
- length = len(strings[0])
57
- totalStrings = len(strings)
58
- strings = [long(string, base=2) for string in strings]
59
- meshes = []
60
- for k in range(attempts):
61
- matched = []
62
- random.shuffle(strings)
63
- dim = len(strings)
64
- for i in range(dim - 2, -2, -2):
65
- num1 = strings[i]
66
- num2 = strings[i + 1]
67
- if num1 & num2 == 0:
68
- matched.append(i)
69
- for x in matched:
70
- meshes.append((strings[x], strings[x + 1]))
71
- for x in matched:
72
- del strings[x + 1]
73
- del strings[x]
74
-
75
- formatstring = "{0:0" + str(length) + "b}"
76
- meshes = [(formatstring.format(num), formatstring.format(num2))
77
- for (num, num2) in meshes]
78
- if display:
79
- print "meshes:"
80
- print meshes
81
- if stdOutput:
82
- return 100 * float(len(meshes)) / totalStrings
83
- return len(meshes)
84
-
85
-
86
- def _greedyMesher(strings, stdOutput=False):
87
- """DEPRECATED
88
- Meshes a list of strings using a greedy first-match technique. Returns
89
- the number of matched pairs after available matches are exhausted."""
90
- s = strings
91
- matched_strings = []
92
- unmatched_strings = []
93
- matched = []
94
- for i in range(len(s)):
95
- for j in range(i + 1, len(s)):
96
- if i not in matched and j not in matched:
97
- num = [int(x) for x in list(s[i])]
98
- num2 = [int(x) for x in list(s[j])]
99
- if np.dot(num, num2) == 0:
100
- matched.append(i)
101
- matched.append(j)
102
- matched_strings += [s[x] for x in matched]
103
- unmatched_strings += [s[x] for x in range(len(s)) if x not in matched]
104
-
105
- if stdOutput == True:
106
- return 100 * len(matched_strings) / (2 * len(strings))
107
- else:
108
- return matched_strings, unmatched_strings
109
-
110
-
111
- def greedyMesher(strings, stdOutput=False, cutoff=None):
112
- length = len(strings)
113
- new_strings = formatStrings(strings)
114
- meshes = []
115
- occupancySort(new_strings)
116
- simpleGreedyTraverse(meshes, new_strings, cutoff)
117
- if stdOutput:
118
- return 100 * len(meshes) / length
119
- else:
120
- return meshes
121
-
122
- # def splitter(strings, length, splitting_string = 0):
123
- # splitting_strings = []
124
- # num_splitters = int(math.log(length,2))+1
125
- # for i in range(1,num_splitters):
126
- # split_string = ""
127
- # for j in range(2**(i-1)):
128
- # split_string = split_string + (("1" * int((length/(2**i)))) + ("0" * (int(length/(2**i)))))
129
- # splitting_strings.append(split_string)
130
- # if splitting_string >= num_splitters-1:
131
- # return bucket1, bucket2
132
- # split = splitting_strings[splitting_string]
133
- # bucket1 = []
134
- # bucket2 = []
135
- # for s in strings:
136
- # diff = hamming(s[0], split)
137
- # if diff < int(length * 0.5):
138
- # bucket1.append(s)
139
- # elif diff == int(length * 0.5):
140
- # if random.randint(0,1):
141
- # bucket1.append(s)
142
- # else:
143
- # bucket2.append(s)
144
- # else:
145
- # bucket2.append(s)
146
- # return bucket1, bucket2
147
- #
148
- # def splitAgain(bucket1, bucket2, length, method):
149
- # try:
150
- # new_bucket1, new_bucket2 = splitter(bucket1+bucket2, length, method)
151
- # except IndexError:
152
- # return bucket1, bucket2
153
- # return new_bucket1, new_bucket2
154
-
155
-
156
- def splittingMesher(strings, attempts, splittingMethod=0, display=False, stdOutput=False, extra=True):
157
- if display:
158
- print "using Splitting Mesher"
159
- length = len(strings[0])
160
- new_strings = formatStrings(strings)
161
- splt = Splitter(length)
162
- bucket1, bucket2 = splt.split(strings=new_strings)
163
-
164
- meshes = []
165
- for k in range(attempts):
166
- # if k == attempts/2:
167
- # print "rebucketing at halfway point"
168
- # print bucket1, bucket2
169
- # bucket1, bucket2 = splt.split(bucket1 = bucket1, bucket2 = bucket2)
170
- random.shuffle(bucket1)
171
- random.shuffle(bucket2)
172
- try:
173
- # print bucket1, bucket2, meshes
174
- done = traverse(meshes, bucket1=bucket1,
175
- bucket2=bucket2, extra=extra)
176
- # print bucket1, bucket2, meshes
177
- # print 'that was round {}'.format(k)
178
- except AssertionError:
179
- # print "rebucketing because one bucket is empty"
180
- bucket1, bucket2 = splt.split(bucket1=bucket1, bucket2=bucket2)
181
- continue
182
- if done:
183
- # print "all done, ending early at attempt {}".format(k)
184
- break
185
- if display:
186
- print "meshes:"
187
- print meshes
188
- if stdOutput:
189
- return 100 * float(len(meshes)) / len(strings)
190
- return len(meshes)
191
-
192
-
193
- def randomSplittingMesher(strings, attempts, display=False, stdOutput=False):
194
- """randomly splits string list into two lists, and then tries to mesh pairs
195
- between the lists. for comparison purposes only, not an actual useful meshing
196
- method."""
197
- if display:
198
- print "using random Splitting Mesher"
199
- bucket1, bucket2 = [], []
200
- length = len(strings[0])
201
- # if splittingMethod == "left":
202
- # splittingString = ("1" * (length/2)) + ("0" * (length/2))
203
- # elif splittingMethod == "checkers":
204
- # splittingString = ("10" * (length/2))
205
- for string in strings:
206
- s = long(string, base=2)
207
- if random.randint(0, 1):
208
- bucket1.append(s)
209
- else:
210
- bucket2.append(s)
211
- formatstring = "{0:0" + str(length) + "b}"
212
- # print "bucket1:"
213
- # print [formatstring.format(item) for item in bucket1]
214
- # print "bucket2:"
215
- # print [formatstring.format(item) for item in bucket2]
216
- # print "\n"
217
- # print "bucket2: {0:08b}\n".format(bucket2)
218
-
219
- meshes = []
220
- for k in range(attempts):
221
- random.shuffle(bucket1)
222
- random.shuffle(bucket2)
223
- # print "shuffles: {},\n{}".format(bucket1, bucket2)
224
- dim = min(len(bucket1), len(bucket2))
225
- if dim == 0:
226
- break
227
- matched = []
228
- if dim == 1:
229
- # print "checking {} and {}".format(bucket1[0], bucket2[0])
230
- num1 = bucket1[0]
231
- num2 = bucket2[0]
232
- if num1 & num2 == 0:
233
- matched.append(0)
234
- for i in range(dim - 1, 0, -1):
235
- # print "checking {} and {}".format(bucket1[i], bucket2[i])
236
- num1 = bucket1[i]
237
- num2 = bucket2[i]
238
- if num1 & num2 == 0:
239
- matched.append(i)
240
- for x in matched:
241
- meshes.append((bucket1[x], bucket2[x]))
242
- for x in matched:
243
- del bucket1[x]
244
- del bucket2[x]
245
- # meshes = [(num.toBinaryString(), num2.toBinaryString()) for (num, num2) in meshes]
246
- meshes = [(formatstring.format(num), formatstring.format(num2))
247
- for (num, num2) in meshes]
248
- if display:
249
- print "meshes:"
250
- print meshes
251
- if stdOutput:
252
- return 100 * float(len(meshes)) / len(strings)
253
- return len(meshes)
254
-
255
-
256
- def greedySplittingMesher(strings, display=False, std_output=True, cutoff=None):
257
- """
258
- Given a list of strings, splits that list into two lists based off
259
- of a distance measure and then exhaustively checks pairs between
260
- the two lists for meshes, greedily taking any it finds. Sorts the
261
- lists in increasing order of occupancy so sparse/sparse meshes are
262
- likely to be discovered. Can specify a cutoff probability below
263
- which potential meshes will not be considered - this saves a lot
264
- of time without affecting performance too much.
265
- """
266
- if display:
267
- print "using greedy splitting mesher"
268
-
269
- length = len(strings[0]) # length of each string, e.g. 4 for '0100'
270
- start = time.time()
271
- new_strings = formatStrings(strings)
272
- splt = Splitter(length)
273
- bucket1, bucket2 = splt.split(strings=new_strings)
274
- # print "preliminaries took {}".format(time.time()-start)
275
-
276
- start = time.time()
277
- meshes = []
278
- # sorts buckets into low -> high occupancy
279
- occupancySort(bucket1)
280
- occupancySort(bucket2)
281
- # print "sorting took {}".format(time.time()-start)
282
-
283
- start = time.time()
284
- done = greedyTraverse(meshes, bucket1=bucket1,
285
- bucket2=bucket2, cutoff=cutoff)
286
- # print "traversal took {}".format(time.time()-start)
287
-
288
- if display:
289
- print "meshes:"
290
- print meshes
291
- if std_output:
292
- return 100 * float(len(meshes)) / len(strings)
293
- else:
294
- return len(meshes)
295
-
296
-
297
- def doubleSplittingMesher(strings, attempts, display=False, stdOutput=False):
298
- """This function is temporary. I will soon merge it with splittingMesher to allow for arbitrary levels of splitting
299
- in the same function."""
300
- if display:
301
- print "using double Splitting Mesher"
302
- buckets = [[], []], [[], []]
303
- length = len(strings[0])
304
- numStrings = len(strings)
305
- splittingString1 = ("1" * (length / 2)) + ("0" * (length / 2))
306
- splittingString2 = ("10" * (length / 2))
307
- for string in strings:
308
- s = long(string, base=2)
309
- diff = hamming(string, splittingString1)
310
- diff2 = hamming(string, splittingString2)
311
-
312
- if diff < int(length * 0.5):
313
- id1 = 0
314
- elif diff == int(length * 0.5):
315
- if random.randint(0, 1):
316
- id1 = 0
317
- else:
318
- id1 = 1
319
- else:
320
- id1 = 1
321
-
322
- if diff2 < int(length * 0.5):
323
- id2 = 0
324
- elif diff == int(length * 0.5):
325
- if random.randint(0, 1):
326
- id2 = 0
327
- else:
328
- id2 = 1
329
- else:
330
- id2 = 1
331
-
332
- buckets[id1][id2].append(s)
333
- formatstring = "{0:0" + str(length) + "b}"
334
- for layer in buckets:
335
- for thing in layer:
336
- print len(thing)
337
- # print buckets
338
-
339
- meshes = []
340
-
341
- check1 = True
342
- check2 = True
343
- for k in range(attempts):
344
- dim1 = min(len(buckets[0][0]), len(buckets[1][1]))
345
- dim2 = min(len(buckets[0][1]), len(buckets[1][0]))
346
- # print dim1, dim2
347
- if dim1 == 0:
348
- if check1:
349
- print 'found meshes for everything in set 1, so stopped after {} attempts'.format(k)
350
- check1 = False
351
- else:
352
- matched1 = []
353
- if dim1 == 1:
354
- num1 = buckets[0][0][0]
355
- num2 = buckets[1][1][0]
356
- if num1 & num2 == 0:
357
- matched1.append(0)
358
- for i in range(dim1 - 1, 0, -1):
359
- num1 = buckets[0][0][i]
360
- num2 = buckets[1][1][i]
361
- if num1 & num2 == 0:
362
- matched1.append(i)
363
- for x in matched1:
364
- meshes.append((buckets[0][0][x], buckets[1][1][x]))
365
- for x in matched1:
366
- del buckets[0][0][x]
367
- del buckets[1][1][x]
368
- if dim2 == 0:
369
- if check2:
370
- print 'found meshes for everything in set 2, so stopped after {} attempts'.format(k)
371
- check2 = False
372
- else:
373
- matched2 = []
374
- if dim2 == 1:
375
- num1 = buckets[0][1][0]
376
- num2 = buckets[1][0][0]
377
- if num1 & num2 == 0:
378
- matched2.append(0)
379
- for i in range(dim2 - 1, 0, -1):
380
- num1 = buckets[0][1][i]
381
- num2 = buckets[1][0][i]
382
- if num1 & num2 == 0:
383
- matched2.append(i)
384
- for x in matched2:
385
- meshes.append((buckets[0][1][x], buckets[1][0][x]))
386
- for x in matched2:
387
- del buckets[0][1][x]
388
- del buckets[1][0][x]
389
-
390
- meshes = [(formatstring.format(num), formatstring.format(num2))
391
- for (num, num2) in meshes]
392
- if display:
393
- print "meshes:"
394
- print meshes
395
- if stdOutput:
396
- return 100 * float(len(meshes)) / len(strings)
397
- return len(meshes)
398
-
399
-
400
- def maxMatchingMesher(strings, stdOutput=False):
401
- """Converts the string set into a meshing graph and finds the maximum matching on said graph."""
402
- graph = makeGraph(strings)
403
- meshes = len(nx.max_weight_matching(graph)) / 2
404
- if stdOutput:
405
- return 100 * float(meshes) / len(strings)
406
- return meshes
407
-
408
-
409
- def color_counter(graph):
410
- """interprets a coloring on a graph as a meshing."""
411
- color = nx.greedy_color(graph)
412
- i = 0
413
- for key, value in color.iteritems():
414
- i = max(i, value)
415
- return i + 1
416
-
417
-
418
- def optimalMesher(strings, stdOutput=False):
419
- """Converts the string set into a meshing graph and finds a greedy coloring on the complement of said graph."""
420
- graph = makeGraph(strings)
421
- graph_c = nx.complement(graph)
422
- meshes = len(strings) - color_counter(graph_c)
423
- if stdOutput:
424
- return 100 * float(meshes) / len(strings)
425
- return meshes
426
-
427
-
428
- def mesherRetrieve(identifier):
429
- fetcher = {"simple": (simpleMesher),
430
- "dumb": (randomMesher),
431
- "greedy": (greedyMesher),
432
- "split": (splittingMesher),
433
- "greedysplit": (greedySplittingMesher),
434
- "doubsplit": (doubleSplittingMesher),
435
- "randsplit": (randomSplittingMesher),
436
- "maxmatch": (maxMatchingMesher),
437
- "color": (optimalMesher)
438
- }
439
- return fetcher[identifier]
440
-
441
-
442
- if __name__ == '__main__':
443
- # print splittingMesher(["00000001", "11111110", "11100000", "00000111"], 10, display = True)
444
- # print splitter([(("1" * 16),0)], 16)
445
- # print greedySplittingMesher(["00000001", "11111110", "11100000", "00000111"], display = True, stdOutput = False)
446
-
447
- # meshes = []
448
- # strings = formatStrings(["00000001", "11111110", "11100000", "00000111"])
449
- # simpleGreedyTraverse(meshes, strings)
450
- # print meshes, strings
451
-
452
- print greedyMesher(["00000001", "11111110", "11100000", "00000111"])