character_set 1.8.0-java → 1.9.0-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +1 -1
- data/CHANGELOG.md +10 -0
- data/README.md +2 -1
- data/Rakefile +12 -5
- data/ext/character_set/character_set.c +78 -67
- data/ext/character_set/java/src/main/java/org/jaynetics/characterset/CharacterSetService.java +15 -0
- data/ext/character_set/java/src/main/java/org/jaynetics/characterset/RubyCharacterSet.java +493 -0
- data/ext/character_set/java/src/main/java/org/jaynetics/characterset/StringOperations.java +138 -0
- data/lib/character_set/character_set.jar +0 -0
- data/lib/character_set/java.rb +6 -0
- data/lib/character_set/parser.rb +5 -3
- data/lib/character_set/predefined_sets/assigned.cps +95 -58
- data/lib/character_set/predefined_sets/emoji.cps +9 -11
- data/lib/character_set/predefined_sets.rb +1 -1
- data/lib/character_set/ruby_fallback/character_set_methods.rb +7 -10
- data/lib/character_set/ruby_fallback/set_methods.rb +215 -52
- data/lib/character_set/ruby_fallback.rb +4 -18
- data/lib/character_set/shared_methods.rb +39 -2
- data/lib/character_set/version.rb +1 -1
- data/lib/character_set/writer.rb +1 -1
- data/lib/character_set.rb +6 -1
- metadata +10 -53
- data/.gitattributes +0 -3
- data/.github/workflows/gouteur.yml +0 -20
- data/.github/workflows/lint.yml +0 -29
- data/.github/workflows/tests.yml +0 -28
- data/.gitignore +0 -33
- data/.gouteur.yml +0 -2
- data/.rspec +0 -3
- data/BENCHMARK.md +0 -90
- data/Gemfile +0 -21
- data/bin/console +0 -21
- data/bin/setup +0 -8
- data/character_set.gemspec +0 -24
- data/ext/character_set/unicode_casefold_table.h +0 -1438
- data/ext/character_set/unicode_casefold_table.h.tmpl +0 -11
- data/lib/character_set/ruby_fallback/vendored_set_classes.rb +0 -385
- data/tasks/benchmark.rake +0 -20
- data/tasks/benchmarks/count_in.rb +0 -13
- data/tasks/benchmarks/cover.rb +0 -25
- data/tasks/benchmarks/delete_in.rb +0 -29
- data/tasks/benchmarks/keep_in.rb +0 -29
- data/tasks/benchmarks/scan.rb +0 -13
- data/tasks/benchmarks/shared.rb +0 -28
- data/tasks/benchmarks/used_by.rb +0 -25
- data/tasks/benchmarks/z_add.rb +0 -12
- data/tasks/benchmarks/z_delete.rb +0 -12
- data/tasks/benchmarks/z_merge.rb +0 -15
- data/tasks/benchmarks/z_minmax.rb +0 -12
- data/tasks/sync_casefold_data.rake +0 -20
- data/tasks/sync_predefined_sets.rake +0 -9
- data/tasks/sync_ruby_spec.rake +0 -65
|
@@ -0,0 +1,493 @@
|
|
|
1
|
+
package org.jaynetics.characterset;
|
|
2
|
+
|
|
3
|
+
import java.util.BitSet;
|
|
4
|
+
import org.jcodings.specific.UTF8Encoding;
|
|
5
|
+
import org.jruby.Ruby;
|
|
6
|
+
import org.jruby.RubyArray;
|
|
7
|
+
import org.jruby.RubyClass;
|
|
8
|
+
import org.jruby.RubyEnumerator;
|
|
9
|
+
import org.jruby.RubyFixnum;
|
|
10
|
+
import org.jruby.RubyObject;
|
|
11
|
+
import org.jruby.RubyRange;
|
|
12
|
+
import org.jruby.RubyString;
|
|
13
|
+
import org.jruby.anno.JRubyMethod;
|
|
14
|
+
import org.jruby.runtime.Block;
|
|
15
|
+
import org.jruby.runtime.ThreadContext;
|
|
16
|
+
import org.jruby.runtime.Visibility;
|
|
17
|
+
import org.jruby.runtime.builtin.IRubyObject;
|
|
18
|
+
|
|
19
|
+
public final class RubyCharacterSet extends RubyObject {
|
|
20
|
+
static final int LIMIT = 0x110000;
|
|
21
|
+
private BitSet bits = new BitSet(128);
|
|
22
|
+
|
|
23
|
+
public RubyCharacterSet(Ruby runtime, RubyClass klass) {
|
|
24
|
+
super(runtime, klass);
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
static int codepoint(ThreadContext context, IRubyObject value) {
|
|
28
|
+
if (value instanceof RubyFixnum) {
|
|
29
|
+
long cp = ((RubyFixnum) value).getLongValue();
|
|
30
|
+
if (cp >= 0 && cp < LIMIT) return (int) cp;
|
|
31
|
+
}
|
|
32
|
+
throw context.runtime.newArgumentError("CharacterSet members must be between 0 and 0x10FFFF");
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
private RubyCharacterSet result(ThreadContext context, BitSet data) {
|
|
36
|
+
RubyCharacterSet set = (RubyCharacterSet) getMetaClass().getRealClass().allocate();
|
|
37
|
+
set.bits = data;
|
|
38
|
+
return set;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
private static RubyCharacterSet construct(ThreadContext context, IRubyObject klass) {
|
|
42
|
+
return (RubyCharacterSet) ((RubyClass) klass).newInstance(context, Block.NULL_BLOCK);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
private static boolean characterSet(ThreadContext context, IRubyObject value) {
|
|
46
|
+
RubyClass klass = context.runtime.getClass("CharacterSet");
|
|
47
|
+
if (klass.isInstance(value)) return true;
|
|
48
|
+
IRubyObject pure = klass.getConstantAt("Pure");
|
|
49
|
+
return pure instanceof RubyClass && ((RubyClass) pure).isInstance(value);
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
private static BitSet operand(ThreadContext context, IRubyObject other) {
|
|
53
|
+
if (other instanceof RubyCharacterSet) return ((RubyCharacterSet) other).bits;
|
|
54
|
+
if (!characterSet(context, other)) throw context.runtime.newArgumentError("pass a CharacterSet");
|
|
55
|
+
BitSet data = new BitSet();
|
|
56
|
+
RubyArray<?> values = (RubyArray<?>) other.callMethod(context, "to_a");
|
|
57
|
+
for (int i = 0; i < values.getLength(); i++) data.set(codepoint(context, values.entry(i)));
|
|
58
|
+
return data;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
private IRubyObject enumerator(ThreadContext context, String method) {
|
|
62
|
+
return RubyEnumerator.enumeratorizeWithSize(context, this, method,
|
|
63
|
+
(ctx, self, args) -> ctx.runtime.newFixnum(((RubyCharacterSet) self).bits.cardinality()));
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
@JRubyMethod(name = {"size", "length"})
|
|
67
|
+
public IRubyObject size(ThreadContext context) {
|
|
68
|
+
return context.runtime.newFixnum(bits.cardinality());
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
@JRubyMethod(name = "empty?")
|
|
72
|
+
public IRubyObject empty(ThreadContext context) {
|
|
73
|
+
return context.runtime.newBoolean(bits.isEmpty());
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
@JRubyMethod(name = {"include?", "member?", "==="})
|
|
77
|
+
public IRubyObject includes(ThreadContext context, IRubyObject value) {
|
|
78
|
+
return context.runtime.newBoolean(bits.get(codepoint(context, value)));
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
@JRubyMethod(name = {"add", "<<"})
|
|
82
|
+
public IRubyObject add(ThreadContext context, IRubyObject value) {
|
|
83
|
+
checkFrozen();
|
|
84
|
+
bits.set(codepoint(context, value));
|
|
85
|
+
return this;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
@JRubyMethod(name = "add?")
|
|
89
|
+
public IRubyObject addIfAbsent(ThreadContext context, IRubyObject value) {
|
|
90
|
+
checkFrozen();
|
|
91
|
+
int cp = codepoint(context, value);
|
|
92
|
+
if (bits.get(cp)) return context.nil;
|
|
93
|
+
bits.set(cp);
|
|
94
|
+
return this;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
@JRubyMethod(name = "delete")
|
|
98
|
+
public IRubyObject delete(ThreadContext context, IRubyObject value) {
|
|
99
|
+
checkFrozen();
|
|
100
|
+
bits.clear(codepoint(context, value));
|
|
101
|
+
return this;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
@JRubyMethod(name = "delete?")
|
|
105
|
+
public IRubyObject deleteIfPresent(ThreadContext context, IRubyObject value) {
|
|
106
|
+
checkFrozen();
|
|
107
|
+
int cp = codepoint(context, value);
|
|
108
|
+
if (!bits.get(cp)) return context.nil;
|
|
109
|
+
bits.clear(cp);
|
|
110
|
+
return this;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
@JRubyMethod(name = "clear")
|
|
114
|
+
public IRubyObject clear(ThreadContext context) {
|
|
115
|
+
checkFrozen();
|
|
116
|
+
bits.clear();
|
|
117
|
+
return this;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
private void mergeRange(ThreadContext context, RubyRange range) {
|
|
121
|
+
int from = codepoint(context, range.begin(context));
|
|
122
|
+
IRubyObject end = range.end(context);
|
|
123
|
+
if (!(end instanceof RubyFixnum)) throw context.runtime.newArgumentError("pass a bounded codepoint Range");
|
|
124
|
+
long upto = ((RubyFixnum) end).getLongValue();
|
|
125
|
+
if (range.isExcludeEnd()) upto--;
|
|
126
|
+
if (upto < -1 || upto >= LIMIT) throw context.runtime.newArgumentError("Range exceeds Unicode codepoints");
|
|
127
|
+
checkFrozen();
|
|
128
|
+
if (upto >= from) bits.set(from, (int) upto + 1);
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
private static boolean stringRange(ThreadContext context, IRubyObject value) {
|
|
132
|
+
if (!(value instanceof RubyRange)) return false;
|
|
133
|
+
RubyRange range = (RubyRange) value;
|
|
134
|
+
return range.begin(context) instanceof RubyString && range.end(context) instanceof RubyString;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
@JRubyMethod(name = "merge")
|
|
138
|
+
public IRubyObject merge(ThreadContext context, IRubyObject other) {
|
|
139
|
+
checkFrozen();
|
|
140
|
+
if (other instanceof RubyCharacterSet) {
|
|
141
|
+
bits.or(((RubyCharacterSet) other).bits);
|
|
142
|
+
} else if (other instanceof RubyRange && !stringRange(context, other)) {
|
|
143
|
+
mergeRange(context, (RubyRange) other);
|
|
144
|
+
} else {
|
|
145
|
+
if (!other.respondsTo("each")) throw context.runtime.newArgumentError("pass an Enumerable");
|
|
146
|
+
RubyArray<?> values = (RubyArray<?>) other.callMethod(context, "to_a");
|
|
147
|
+
checkFrozen();
|
|
148
|
+
for (int i = 0; i < values.getLength(); i++) {
|
|
149
|
+
IRubyObject value = values.entry(i);
|
|
150
|
+
if (value instanceof RubyString) value = value.callMethod(context, "ord");
|
|
151
|
+
int cp = codepoint(context, value);
|
|
152
|
+
checkFrozen();
|
|
153
|
+
bits.set(cp);
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
return this;
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
@JRubyMethod(name = "subtract")
|
|
160
|
+
public IRubyObject subtract(ThreadContext context, IRubyObject other) {
|
|
161
|
+
checkFrozen();
|
|
162
|
+
BitSet data = operand(context, other);
|
|
163
|
+
checkFrozen();
|
|
164
|
+
bits.andNot(data);
|
|
165
|
+
return this;
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
@JRubyMethod(name = "initialize_copy", visibility = Visibility.PRIVATE)
|
|
169
|
+
public IRubyObject copy(ThreadContext context, IRubyObject source) {
|
|
170
|
+
checkFrozen();
|
|
171
|
+
BitSet data = operand(context, source);
|
|
172
|
+
checkFrozen();
|
|
173
|
+
bits = (BitSet) data.clone();
|
|
174
|
+
return this;
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
@JRubyMethod(name = {"union", "+", "|"})
|
|
178
|
+
public IRubyObject union(ThreadContext context, IRubyObject other) {
|
|
179
|
+
BitSet data = (BitSet) bits.clone();
|
|
180
|
+
data.or(operand(context, other));
|
|
181
|
+
return result(context, data);
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
@JRubyMethod(name = {"intersection", "&"})
|
|
185
|
+
public IRubyObject intersection(ThreadContext context, IRubyObject other) {
|
|
186
|
+
BitSet data = (BitSet) bits.clone();
|
|
187
|
+
data.and(operand(context, other));
|
|
188
|
+
return result(context, data);
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
@JRubyMethod(name = {"difference", "-"})
|
|
192
|
+
public IRubyObject difference(ThreadContext context, IRubyObject other) {
|
|
193
|
+
BitSet data = (BitSet) bits.clone();
|
|
194
|
+
data.andNot(operand(context, other));
|
|
195
|
+
return result(context, data);
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
@JRubyMethod(name = "^")
|
|
199
|
+
public IRubyObject exclusion(ThreadContext context, IRubyObject other) {
|
|
200
|
+
BitSet data = (BitSet) bits.clone();
|
|
201
|
+
data.xor(operand(context, other));
|
|
202
|
+
return result(context, data);
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
@JRubyMethod(name = "intersect?")
|
|
206
|
+
public IRubyObject intersects(ThreadContext context, IRubyObject other) {
|
|
207
|
+
return context.runtime.newBoolean(bits.intersects(operand(context, other)));
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
@JRubyMethod(name = "disjoint?")
|
|
211
|
+
public IRubyObject disjoint(ThreadContext context, IRubyObject other) {
|
|
212
|
+
return context.runtime.newBoolean(!bits.intersects(operand(context, other)));
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
private static boolean subset(BitSet a, BitSet b) {
|
|
216
|
+
BitSet remainder = (BitSet) a.clone();
|
|
217
|
+
remainder.andNot(b);
|
|
218
|
+
return remainder.isEmpty();
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
@JRubyMethod(name = {"subset?", "<="})
|
|
222
|
+
public IRubyObject subsetOf(ThreadContext context, IRubyObject other) {
|
|
223
|
+
return context.runtime.newBoolean(subset(bits, operand(context, other)));
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
@JRubyMethod(name = {"proper_subset?", "<"})
|
|
227
|
+
public IRubyObject properSubsetOf(ThreadContext context, IRubyObject other) {
|
|
228
|
+
BitSet data = operand(context, other);
|
|
229
|
+
return context.runtime.newBoolean(!bits.equals(data) && subset(bits, data));
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
@JRubyMethod(name = {"superset?", ">="})
|
|
233
|
+
public IRubyObject supersetOf(ThreadContext context, IRubyObject other) {
|
|
234
|
+
return context.runtime.newBoolean(subset(operand(context, other), bits));
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
@JRubyMethod(name = {"proper_superset?", ">"})
|
|
238
|
+
public IRubyObject properSupersetOf(ThreadContext context, IRubyObject other) {
|
|
239
|
+
BitSet data = operand(context, other);
|
|
240
|
+
return context.runtime.newBoolean(!bits.equals(data) && subset(data, bits));
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
@JRubyMethod(name = "==")
|
|
244
|
+
public IRubyObject equalValue(ThreadContext context, IRubyObject other) {
|
|
245
|
+
return context.runtime.newBoolean(characterSet(context, other) && bits.equals(operand(context, other)));
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
@JRubyMethod(name = "eql?")
|
|
249
|
+
public IRubyObject eql(ThreadContext context, IRubyObject other) {
|
|
250
|
+
return context.runtime.newBoolean(other instanceof RubyCharacterSet
|
|
251
|
+
&& bits.equals(((RubyCharacterSet) other).bits));
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
@JRubyMethod(name = "hash")
|
|
255
|
+
public IRubyObject contentHash(ThreadContext context) {
|
|
256
|
+
return context.runtime.newFixnum(bits.hashCode());
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
@JRubyMethod(name = "<=>")
|
|
260
|
+
public IRubyObject compare(ThreadContext context, IRubyObject other) {
|
|
261
|
+
if (!characterSet(context, other)) return context.nil;
|
|
262
|
+
BitSet data = operand(context, other);
|
|
263
|
+
if (bits.equals(data)) return context.runtime.newFixnum(0);
|
|
264
|
+
if (subset(bits, data)) return context.runtime.newFixnum(-1);
|
|
265
|
+
if (subset(data, bits)) return context.runtime.newFixnum(1);
|
|
266
|
+
return context.nil;
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
@JRubyMethod(name = "each")
|
|
270
|
+
public IRubyObject each(ThreadContext context, Block block) {
|
|
271
|
+
if (!block.isGiven()) return enumerator(context, "each");
|
|
272
|
+
// Snapshot iteration permits mutation during Ruby callbacks without sharing state.
|
|
273
|
+
BitSet snapshot = (BitSet) bits.clone();
|
|
274
|
+
for (int cp = snapshot.nextSetBit(0); cp >= 0; cp = snapshot.nextSetBit(cp + 1)) {
|
|
275
|
+
block.yield(context, context.runtime.newFixnum(cp));
|
|
276
|
+
}
|
|
277
|
+
return this;
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
private IRubyObject filter(ThreadContext context, Block block, boolean keep) {
|
|
281
|
+
if (!block.isGiven()) return enumerator(context, keep ? "keep_if" : "delete_if");
|
|
282
|
+
checkFrozen();
|
|
283
|
+
BitSet snapshot = (BitSet) bits.clone();
|
|
284
|
+
for (int cp = snapshot.nextSetBit(0); cp >= 0; cp = snapshot.nextSetBit(cp + 1)) {
|
|
285
|
+
boolean accepted = block.yield(context, context.runtime.newFixnum(cp)).isTrue();
|
|
286
|
+
if (accepted != keep) {
|
|
287
|
+
checkFrozen();
|
|
288
|
+
bits.clear(cp);
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
return this;
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
@JRubyMethod(name = "keep_if")
|
|
295
|
+
public IRubyObject keepIf(ThreadContext context, Block block) {
|
|
296
|
+
return filter(context, block, true);
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
@JRubyMethod(name = "delete_if")
|
|
300
|
+
public IRubyObject deleteIf(ThreadContext context, Block block) {
|
|
301
|
+
return filter(context, block, false);
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
@JRubyMethod(name = "to_a", optional = 1)
|
|
305
|
+
public IRubyObject toArray(ThreadContext context, IRubyObject[] args) {
|
|
306
|
+
RubyArray<?> array = context.runtime.newArray(bits.cardinality());
|
|
307
|
+
boolean stringify = args.length > 0 && args[0].isTrue();
|
|
308
|
+
for (int cp = bits.nextSetBit(0); cp >= 0; cp = bits.nextSetBit(cp + 1)) {
|
|
309
|
+
array.append(stringify ? StringOperations.character(context, cp) : context.runtime.newFixnum(cp));
|
|
310
|
+
}
|
|
311
|
+
return array;
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
@JRubyMethod(name = "min")
|
|
315
|
+
public IRubyObject min(ThreadContext context) {
|
|
316
|
+
int cp = bits.nextSetBit(0);
|
|
317
|
+
return cp < 0 ? context.nil : context.runtime.newFixnum(cp);
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
@JRubyMethod(name = "max")
|
|
321
|
+
public IRubyObject max(ThreadContext context) {
|
|
322
|
+
int cp = bits.length() - 1;
|
|
323
|
+
return cp < 0 ? context.nil : context.runtime.newFixnum(cp);
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
@JRubyMethod(name = "minmax")
|
|
327
|
+
public IRubyObject minmax(ThreadContext context) {
|
|
328
|
+
return context.runtime.newArray(min(context), max(context));
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
@JRubyMethod(name = "ranges")
|
|
332
|
+
public IRubyObject ranges(ThreadContext context) {
|
|
333
|
+
RubyArray<?> array = context.runtime.newArray();
|
|
334
|
+
for (int from = bits.nextSetBit(0); from >= 0;) {
|
|
335
|
+
int end = bits.nextClearBit(from);
|
|
336
|
+
array.append(RubyRange.newRange(context,
|
|
337
|
+
context.runtime.newFixnum(from), context.runtime.newFixnum(end - 1), false));
|
|
338
|
+
from = bits.nextSetBit(end);
|
|
339
|
+
}
|
|
340
|
+
return array;
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
@JRubyMethod(name = "sample", optional = 1)
|
|
344
|
+
public IRubyObject sample(ThreadContext context, IRubyObject[] args) {
|
|
345
|
+
IRubyObject array = toArray(context, new IRubyObject[] {context.tru});
|
|
346
|
+
return array.callMethod(context, "sample", args);
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
@JRubyMethod(name = "from_ranges", meta = true, rest = true)
|
|
350
|
+
public static IRubyObject fromRanges(ThreadContext context, IRubyObject klass, IRubyObject[] ranges) {
|
|
351
|
+
RubyCharacterSet set = construct(context, klass);
|
|
352
|
+
for (IRubyObject range : ranges) {
|
|
353
|
+
if (!(range instanceof RubyRange)) throw context.runtime.newArgumentError("pass a Range");
|
|
354
|
+
set.mergeRange(context, (RubyRange) range);
|
|
355
|
+
}
|
|
356
|
+
return set;
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
@JRubyMethod(name = "ext_section", required = 2)
|
|
360
|
+
public IRubyObject section(ThreadContext context, IRubyObject from, IRubyObject upto) {
|
|
361
|
+
int start = codepoint(context, from);
|
|
362
|
+
int end = codepoint(context, upto);
|
|
363
|
+
BitSet data = (BitSet) bits.clone();
|
|
364
|
+
data.clear(0, start);
|
|
365
|
+
data.clear(end + 1, Math.max(end + 1, data.length()));
|
|
366
|
+
return result(context, data);
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
private int sectionCount(ThreadContext context, IRubyObject from, IRubyObject upto) {
|
|
370
|
+
int start = codepoint(context, from);
|
|
371
|
+
int end = codepoint(context, upto);
|
|
372
|
+
return start > end ? 0 : bits.get(start, end + 1).cardinality();
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
@JRubyMethod(name = "ext_count_in_section", required = 2)
|
|
376
|
+
public IRubyObject countSection(ThreadContext context, IRubyObject from, IRubyObject upto) {
|
|
377
|
+
return context.runtime.newFixnum(sectionCount(context, from, upto));
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
@JRubyMethod(name = "ext_section?", required = 2)
|
|
381
|
+
public IRubyObject hasSection(ThreadContext context, IRubyObject from, IRubyObject upto) {
|
|
382
|
+
int start = codepoint(context, from);
|
|
383
|
+
int end = codepoint(context, upto);
|
|
384
|
+
int first = bits.nextSetBit(start);
|
|
385
|
+
return context.runtime.newBoolean(first >= 0 && first <= end);
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
@JRubyMethod(name = "ext_section_ratio", required = 2)
|
|
389
|
+
public IRubyObject sectionRatio(ThreadContext context, IRubyObject from, IRubyObject upto) {
|
|
390
|
+
return context.runtime.newFloat((double) sectionCount(context, from, upto) / bits.cardinality());
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
@JRubyMethod(name = "planes")
|
|
394
|
+
public IRubyObject planes(ThreadContext context) {
|
|
395
|
+
RubyArray<?> array = context.runtime.newArray();
|
|
396
|
+
for (int cp = bits.nextSetBit(0); cp >= 0;) {
|
|
397
|
+
int plane = cp >>> 16;
|
|
398
|
+
array.append(context.runtime.newFixnum(plane));
|
|
399
|
+
cp = bits.nextSetBit((plane + 1) << 16);
|
|
400
|
+
}
|
|
401
|
+
return array;
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
private static int planeNumber(ThreadContext context, IRubyObject number) {
|
|
405
|
+
int plane = codepoint(context, number);
|
|
406
|
+
if (plane > 16) throw context.runtime.newArgumentError("plane must be between 0 and 16");
|
|
407
|
+
return plane;
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
@JRubyMethod(name = "plane")
|
|
411
|
+
public IRubyObject plane(ThreadContext context, IRubyObject number) {
|
|
412
|
+
int start = planeNumber(context, number) << 16;
|
|
413
|
+
return section(context, context.runtime.newFixnum(start), context.runtime.newFixnum(start + 0xFFFF));
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
@JRubyMethod(name = "member_in_plane?")
|
|
417
|
+
public IRubyObject hasPlane(ThreadContext context, IRubyObject number) {
|
|
418
|
+
int start = planeNumber(context, number) << 16;
|
|
419
|
+
int first = bits.nextSetBit(start);
|
|
420
|
+
return context.runtime.newBoolean(first >= 0 && first < start + 0x10000);
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
@JRubyMethod(name = "ext_inversion", required = 2)
|
|
424
|
+
public IRubyObject inversion(ThreadContext context, IRubyObject includeSurrogates, IRubyObject upto) {
|
|
425
|
+
int end = codepoint(context, upto);
|
|
426
|
+
BitSet data = bits.get(0, end + 1);
|
|
427
|
+
data.flip(0, end + 1);
|
|
428
|
+
if (!includeSurrogates.isTrue()) data.clear(0xD800, 0xE000);
|
|
429
|
+
return result(context, data);
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
@JRubyMethod(name = "case_insensitive")
|
|
433
|
+
public IRubyObject caseInsensitive(ThreadContext context) {
|
|
434
|
+
BitSet data = (BitSet) bits.clone();
|
|
435
|
+
// Flags 0 exclude multi-character folds; the API supplies both directions.
|
|
436
|
+
UTF8Encoding.INSTANCE.applyAllCaseFold(0, (from, to, length, ignored) -> {
|
|
437
|
+
if (length == 1 && bits.get(from)) data.set(to[0]);
|
|
438
|
+
}, null);
|
|
439
|
+
return result(context, data);
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
@JRubyMethod(name = "allocated_length")
|
|
443
|
+
public IRubyObject allocatedLength(ThreadContext context) {
|
|
444
|
+
return context.runtime.newFixnum(bits.size());
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
@JRubyMethod(name = "of_string", meta = true)
|
|
448
|
+
public static IRubyObject ofString(ThreadContext context, IRubyObject klass, IRubyObject string) {
|
|
449
|
+
RubyCharacterSet set = construct(context, klass);
|
|
450
|
+
StringOperations.populate(context, string, set.bits);
|
|
451
|
+
return set;
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
@JRubyMethod(name = "count_in")
|
|
455
|
+
public IRubyObject countIn(ThreadContext context, IRubyObject string) {
|
|
456
|
+
return StringOperations.query(context, string, bits, StringOperations.COUNT);
|
|
457
|
+
}
|
|
458
|
+
|
|
459
|
+
@JRubyMethod(name = "cover?")
|
|
460
|
+
public IRubyObject covers(ThreadContext context, IRubyObject string) {
|
|
461
|
+
return StringOperations.query(context, string, bits, StringOperations.COVER);
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
@JRubyMethod(name = "used_by?")
|
|
465
|
+
public IRubyObject usedBy(ThreadContext context, IRubyObject string) {
|
|
466
|
+
return StringOperations.query(context, string, bits, StringOperations.ANY);
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
@JRubyMethod(name = "scan")
|
|
470
|
+
public IRubyObject scan(ThreadContext context, IRubyObject string) {
|
|
471
|
+
return StringOperations.scan(context, string, bits);
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
@JRubyMethod(name = "delete_in")
|
|
475
|
+
public IRubyObject deleteIn(ThreadContext context, IRubyObject string) {
|
|
476
|
+
return StringOperations.filter(context, string, bits, false, false);
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
@JRubyMethod(name = "delete_in!")
|
|
480
|
+
public IRubyObject deleteInPlace(ThreadContext context, IRubyObject string) {
|
|
481
|
+
return StringOperations.filter(context, string, bits, false, true);
|
|
482
|
+
}
|
|
483
|
+
|
|
484
|
+
@JRubyMethod(name = "keep_in")
|
|
485
|
+
public IRubyObject keepIn(ThreadContext context, IRubyObject string) {
|
|
486
|
+
return StringOperations.filter(context, string, bits, true, false);
|
|
487
|
+
}
|
|
488
|
+
|
|
489
|
+
@JRubyMethod(name = "keep_in!")
|
|
490
|
+
public IRubyObject keepInPlace(ThreadContext context, IRubyObject string) {
|
|
491
|
+
return StringOperations.filter(context, string, bits, true, true);
|
|
492
|
+
}
|
|
493
|
+
}
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
package org.jaynetics.characterset;
|
|
2
|
+
|
|
3
|
+
import java.util.Arrays;
|
|
4
|
+
import java.util.BitSet;
|
|
5
|
+
import org.jcodings.Encoding;
|
|
6
|
+
import org.jcodings.specific.ASCIIEncoding;
|
|
7
|
+
import org.jcodings.specific.UTF8Encoding;
|
|
8
|
+
import org.jruby.RubyArray;
|
|
9
|
+
import org.jruby.RubyString;
|
|
10
|
+
import org.jruby.runtime.ThreadContext;
|
|
11
|
+
import org.jruby.runtime.builtin.IRubyObject;
|
|
12
|
+
import org.jruby.util.ByteList;
|
|
13
|
+
|
|
14
|
+
final class StringOperations {
|
|
15
|
+
static final int COUNT = 0;
|
|
16
|
+
static final int COVER = 1;
|
|
17
|
+
static final int ANY = 2;
|
|
18
|
+
|
|
19
|
+
private static RubyString string(ThreadContext context, IRubyObject value) {
|
|
20
|
+
if (!(value instanceof RubyString)) throw context.runtime.newArgumentError("pass a String");
|
|
21
|
+
return (RubyString) value;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
private static RubyString unicode(ThreadContext context, RubyString string) {
|
|
25
|
+
Encoding encoding = string.getEncoding();
|
|
26
|
+
if (encoding == UTF8Encoding.INSTANCE || encoding == ASCIIEncoding.INSTANCE
|
|
27
|
+
|| (encoding.isAsciiCompatible() && string.isAsciiOnly())) return string;
|
|
28
|
+
return (RubyString) string.encode(context,
|
|
29
|
+
context.runtime.getEncodingService().getEncoding(UTF8Encoding.INSTANCE));
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
private static int width(ThreadContext context, byte[] bytes, int position, int end, boolean binary) {
|
|
33
|
+
if (binary || bytes[position] >= 0) return 1;
|
|
34
|
+
int length = UTF8Encoding.INSTANCE.length(bytes, position, end);
|
|
35
|
+
if (length <= 0) throw context.runtime.newArgumentError("invalid byte sequence in UTF-8");
|
|
36
|
+
return length;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
private static int codepoint(byte[] bytes, int position, int length) {
|
|
40
|
+
if (length == 1) return bytes[position] & 255;
|
|
41
|
+
return UTF8Encoding.INSTANCE.mbcToCode(bytes, position, position + length);
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
static RubyString character(ThreadContext context, int cp) {
|
|
45
|
+
// Delegate UTF-8 character construction, including surrogate policy, to Ruby.
|
|
46
|
+
return (RubyString) context.runtime.newFixnum(cp).chr(context,
|
|
47
|
+
context.runtime.getEncodingService().getEncoding(UTF8Encoding.INSTANCE));
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
static void populate(ThreadContext context, IRubyObject value, BitSet bits) {
|
|
51
|
+
RubyString input = unicode(context, string(context, value));
|
|
52
|
+
ByteList data = input.getByteList();
|
|
53
|
+
byte[] bytes = data.getUnsafeBytes();
|
|
54
|
+
int end = data.getBegin() + data.realSize();
|
|
55
|
+
boolean binary = input.getEncoding() == ASCIIEncoding.INSTANCE;
|
|
56
|
+
for (int p = data.getBegin(); p < end;) {
|
|
57
|
+
int length = width(context, bytes, p, end, binary);
|
|
58
|
+
bits.set(codepoint(bytes, p, length));
|
|
59
|
+
p += length;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
static IRubyObject query(ThreadContext context, IRubyObject value, BitSet bits, int operation) {
|
|
64
|
+
RubyString input = unicode(context, string(context, value));
|
|
65
|
+
ByteList data = input.getByteList();
|
|
66
|
+
byte[] bytes = data.getUnsafeBytes();
|
|
67
|
+
int end = data.getBegin() + data.realSize();
|
|
68
|
+
boolean binary = input.getEncoding() == ASCIIEncoding.INSTANCE;
|
|
69
|
+
int count = 0;
|
|
70
|
+
for (int p = data.getBegin(); p < end;) {
|
|
71
|
+
int length = width(context, bytes, p, end, binary);
|
|
72
|
+
boolean found = bits.get(codepoint(bytes, p, length));
|
|
73
|
+
if (operation == COVER && !found) return context.fals;
|
|
74
|
+
if (operation == ANY && found) return context.tru;
|
|
75
|
+
if (found) count++;
|
|
76
|
+
p += length;
|
|
77
|
+
}
|
|
78
|
+
if (operation == COUNT) return context.runtime.newFixnum(count);
|
|
79
|
+
return context.runtime.newBoolean(operation == COVER);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
static IRubyObject scan(ThreadContext context, IRubyObject value, BitSet bits) {
|
|
83
|
+
RubyString input = unicode(context, string(context, value));
|
|
84
|
+
ByteList data = input.getByteList();
|
|
85
|
+
byte[] bytes = data.getUnsafeBytes();
|
|
86
|
+
int end = data.getBegin() + data.realSize();
|
|
87
|
+
boolean binary = input.getEncoding() == ASCIIEncoding.INSTANCE;
|
|
88
|
+
RubyArray<?> result = context.runtime.newArray();
|
|
89
|
+
for (int p = data.getBegin(); p < end;) {
|
|
90
|
+
int length = width(context, bytes, p, end, binary);
|
|
91
|
+
int cp = codepoint(bytes, p, length);
|
|
92
|
+
if (bits.get(cp)) {
|
|
93
|
+
result.append(binary ? character(context, cp) : RubyString.newStringNoCopy(context.runtime,
|
|
94
|
+
Arrays.copyOfRange(bytes, p, p + length), 0, length, UTF8Encoding.INSTANCE));
|
|
95
|
+
}
|
|
96
|
+
p += length;
|
|
97
|
+
}
|
|
98
|
+
return result;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
static IRubyObject filter(ThreadContext context, IRubyObject value, BitSet bits, boolean keep, boolean bang) {
|
|
102
|
+
RubyString original = string(context, value);
|
|
103
|
+
if (bang) original.modifyCheck();
|
|
104
|
+
RubyString input = unicode(context, original);
|
|
105
|
+
ByteList data = input.getByteList();
|
|
106
|
+
byte[] bytes = data.getUnsafeBytes();
|
|
107
|
+
int begin = data.getBegin();
|
|
108
|
+
int end = begin + data.realSize();
|
|
109
|
+
boolean binary = input.getEncoding() == ASCIIEncoding.INSTANCE;
|
|
110
|
+
byte[] output = null;
|
|
111
|
+
int written = 0;
|
|
112
|
+
int span = begin;
|
|
113
|
+
for (int p = begin; p < end;) {
|
|
114
|
+
int length = width(context, bytes, p, end, binary);
|
|
115
|
+
if (bits.get(codepoint(bytes, p, length)) != keep) {
|
|
116
|
+
if (output == null) output = new byte[data.realSize()];
|
|
117
|
+
int retained = p - span;
|
|
118
|
+
System.arraycopy(bytes, span, output, written, retained);
|
|
119
|
+
written += retained;
|
|
120
|
+
span = p + length;
|
|
121
|
+
}
|
|
122
|
+
p += length;
|
|
123
|
+
}
|
|
124
|
+
if (output == null) return bang ? context.nil : original.strDup(context.runtime);
|
|
125
|
+
System.arraycopy(bytes, span, output, written, end - span);
|
|
126
|
+
written += end - span;
|
|
127
|
+
RubyString result = RubyString.newStringNoCopy(context.runtime, output, 0, written, input.getEncoding());
|
|
128
|
+
if (input.getEncoding() != original.getEncoding()) {
|
|
129
|
+
result = (RubyString) result.encode(context,
|
|
130
|
+
context.runtime.getEncodingService().getEncoding(original.getEncoding()));
|
|
131
|
+
}
|
|
132
|
+
if (!bang) return result;
|
|
133
|
+
original.replace(result);
|
|
134
|
+
return original;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
private StringOperations() {}
|
|
138
|
+
}
|
|
Binary file
|
data/lib/character_set/parser.rb
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
class CharacterSet
|
|
2
4
|
module Parser
|
|
3
5
|
module_function
|
|
@@ -18,7 +20,7 @@ class CharacterSet
|
|
|
18
20
|
|
|
19
21
|
def codepoints_from_bracket_expression(string)
|
|
20
22
|
raise ArgumentError, 'pass a String' unless string.is_a?(String)
|
|
21
|
-
raise ArgumentError, 'advanced syntax' if string
|
|
23
|
+
raise ArgumentError, 'advanced syntax' if string.match?(/\\[^uUx]|[^\\]\[|&&/)
|
|
22
24
|
|
|
23
25
|
content = strip_brackets(string)
|
|
24
26
|
literal_content = eval_escapes(content)
|
|
@@ -26,7 +28,7 @@ class CharacterSet
|
|
|
26
28
|
prev_chr = nil
|
|
27
29
|
in_range = false
|
|
28
30
|
|
|
29
|
-
literal_content.each_char.
|
|
31
|
+
literal_content.each_char.filter_map do |chr|
|
|
30
32
|
if chr == '-' && prev_chr && prev_chr != '\\' && prev_chr != '-'
|
|
31
33
|
in_range = true
|
|
32
34
|
nil
|
|
@@ -36,7 +38,7 @@ class CharacterSet
|
|
|
36
38
|
prev_chr = chr
|
|
37
39
|
result
|
|
38
40
|
end
|
|
39
|
-
end.
|
|
41
|
+
end.flatten
|
|
40
42
|
end
|
|
41
43
|
|
|
42
44
|
def strip_brackets(string)
|