obfs 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +602 -0
- data/lib/obfs/levenshtein.rb +41 -0
- data/lib/obfs/store.rb +5 -37
- data/lib/obfs/string_similarity.rb +41 -0
- data/lib/obfs.rb +2 -4
- metadata +7 -5
- data/lib/text/levenshtein.rb +0 -166
- data/lib/text/white_similarity.rb +0 -66
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 2a4b8b313bcfac3893da631ea24c90b5cf39f7337e4dd804e922edfa893d4d77
|
|
4
|
+
data.tar.gz: b7afaac2c2fa61e82e888223dd6473a14dbde609dea0eb67b06d2069bafb164a
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: b782fbfd5d5c740c3256a74b0fb66083d102cc71c59439c673c9a86bc516a438aa63f673d7a609d55d3376fb548421d846345142429b211eeb3e3b304b0edbc6
|
|
7
|
+
data.tar.gz: 3fdd856e0c0c4d65b593f47893c3a85d8d2a3514a03f28656eebbece06aa5ff22a746c58f9b022b14d761187b0cd5ef7336113912caf82c06c80de4a4881fdce
|
data/README.md
ADDED
|
@@ -0,0 +1,602 @@
|
|
|
1
|
+
# OBFS
|
|
2
|
+
|
|
3
|
+
[](https://badge.fury.io/rb/obfs)
|
|
4
|
+
|
|
5
|
+
File-based, object-oriented data store for Ruby.
|
|
6
|
+
|
|
7
|
+
OBFS maps recursive Ruby object access directly to the filesystem, allowing data to be addressed using dot notation, bracket notation, or a combination of both.
|
|
8
|
+
|
|
9
|
+
```ruby
|
|
10
|
+
datastore.users.cat.preferences.theme = "dark"
|
|
11
|
+
|
|
12
|
+
datastore["users"]["cat"]["preferences"]["theme"]
|
|
13
|
+
# => "dark"
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
The resulting filesystem structure is:
|
|
17
|
+
|
|
18
|
+
```text
|
|
19
|
+
users/
|
|
20
|
+
└── cat/
|
|
21
|
+
└── preferences/
|
|
22
|
+
└── theme
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
OBFS also includes filesystem discovery helpers, including direct existence checks, recursive fuzzy directory search, timestamps, and directory indexing. It uses Ruby standard libraries and independent OBFS-owned text algorithms, with no external runtime gem dependencies.
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
## Quickstart
|
|
29
|
+
|
|
30
|
+
Add OBFS to your `Gemfile`:
|
|
31
|
+
|
|
32
|
+
```ruby
|
|
33
|
+
gem "obfs"
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
Require it:
|
|
37
|
+
|
|
38
|
+
```ruby
|
|
39
|
+
require "obfs"
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
Create a store:
|
|
43
|
+
|
|
44
|
+
```ruby
|
|
45
|
+
datastore = OBFS::Store.new
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
Write data:
|
|
49
|
+
|
|
50
|
+
```ruby
|
|
51
|
+
datastore.somefile = "some value"
|
|
52
|
+
|
|
53
|
+
datastore["somefile"] = "some value"
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
Read data:
|
|
57
|
+
|
|
58
|
+
```ruby
|
|
59
|
+
datastore.somefile
|
|
60
|
+
|
|
61
|
+
datastore["somefile"]
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
Delete data:
|
|
65
|
+
|
|
66
|
+
```ruby
|
|
67
|
+
datastore.somefile = nil
|
|
68
|
+
|
|
69
|
+
datastore["somefile"] = nil
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
Supported values include:
|
|
73
|
+
|
|
74
|
+
- String
|
|
75
|
+
- Array
|
|
76
|
+
- Hash
|
|
77
|
+
- Integer
|
|
78
|
+
- Float
|
|
79
|
+
- Boolean
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
## Usage
|
|
83
|
+
|
|
84
|
+
By default, OBFS stores data inside:
|
|
85
|
+
|
|
86
|
+
```text
|
|
87
|
+
~/.obfs
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
A custom path can be specified:
|
|
91
|
+
|
|
92
|
+
```ruby
|
|
93
|
+
datastore = OBFS::Store.new({
|
|
94
|
+
path: "/some/other/folder"
|
|
95
|
+
})
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
Paths are recursively created as needed.
|
|
99
|
+
|
|
100
|
+
For example:
|
|
101
|
+
|
|
102
|
+
```ruby
|
|
103
|
+
datastore.some.long.path.to.create = "some string"
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
creates:
|
|
107
|
+
|
|
108
|
+
```text
|
|
109
|
+
some/
|
|
110
|
+
└── long/
|
|
111
|
+
└── path/
|
|
112
|
+
└── to/
|
|
113
|
+
└── create
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
The file `create` contains the JSON-serialized value:
|
|
117
|
+
|
|
118
|
+
```text
|
|
119
|
+
"some string"
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
Bracket notation can also be used:
|
|
123
|
+
|
|
124
|
+
```ruby
|
|
125
|
+
datastore["some"]["long"]["path"]["to"]["create"] = "some string"
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
Dot and bracket notation may be mixed:
|
|
129
|
+
|
|
130
|
+
```ruby
|
|
131
|
+
datastore.some["long"].path["to"].create = "some string"
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
Hashes and arrays are serialized as JSON values into leaf files:
|
|
135
|
+
|
|
136
|
+
```ruby
|
|
137
|
+
datastore.settings = {
|
|
138
|
+
"theme" => "dark",
|
|
139
|
+
"notifications" => true
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
datastore.numbers = [1, 2, 3]
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
Reading those values returns the corresponding Ruby objects:
|
|
146
|
+
|
|
147
|
+
```ruby
|
|
148
|
+
datastore.settings
|
|
149
|
+
# => {"theme"=>"dark", "notifications"=>true}
|
|
150
|
+
|
|
151
|
+
datastore.numbers
|
|
152
|
+
# => [1, 2, 3]
|
|
153
|
+
```
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
## Path Safety
|
|
157
|
+
|
|
158
|
+
Each OBFS key represents exactly one filesystem node.
|
|
159
|
+
|
|
160
|
+
Nested paths should therefore be expressed recursively:
|
|
161
|
+
|
|
162
|
+
```ruby
|
|
163
|
+
datastore["foo"]["bar"]
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
rather than by embedding filesystem separators:
|
|
167
|
+
|
|
168
|
+
```ruby
|
|
169
|
+
datastore["foo/bar"]
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
OBFS rejects keys containing filesystem path separators, null bytes, `.` or `..`.
|
|
173
|
+
|
|
174
|
+
For example, the following are rejected:
|
|
175
|
+
|
|
176
|
+
```ruby
|
|
177
|
+
datastore[".."]
|
|
178
|
+
datastore["../foo"]
|
|
179
|
+
datastore["../../etc/passwd"]
|
|
180
|
+
datastore["foo/bar"]
|
|
181
|
+
datastore["..\\Windows"]
|
|
182
|
+
```
|
|
183
|
+
|
|
184
|
+
This prevents dynamic OBFS keys from traversing outside the intended datastore hierarchy.
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
## Special Methods
|
|
188
|
+
|
|
189
|
+
### `_path`
|
|
190
|
+
|
|
191
|
+
Returns the current filesystem path.
|
|
192
|
+
|
|
193
|
+
```ruby
|
|
194
|
+
datastore.some.data._path
|
|
195
|
+
```
|
|
196
|
+
|
|
197
|
+
Example:
|
|
198
|
+
|
|
199
|
+
```text
|
|
200
|
+
/home/user/.obfs/some/data
|
|
201
|
+
```
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
### `_index`
|
|
205
|
+
|
|
206
|
+
Returns the entries in the current directory.
|
|
207
|
+
|
|
208
|
+
```ruby
|
|
209
|
+
datastore.some.data._index
|
|
210
|
+
```
|
|
211
|
+
|
|
212
|
+
Example:
|
|
213
|
+
|
|
214
|
+
```ruby
|
|
215
|
+
[
|
|
216
|
+
"foo",
|
|
217
|
+
"bar",
|
|
218
|
+
"baz"
|
|
219
|
+
]
|
|
220
|
+
```
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
### `_keys`
|
|
224
|
+
|
|
225
|
+
Alias for `_index`.
|
|
226
|
+
|
|
227
|
+
```ruby
|
|
228
|
+
datastore.some.data._keys
|
|
229
|
+
```
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
### `_find`
|
|
233
|
+
|
|
234
|
+
Performs a fuzzy search against entries in the current directory only.
|
|
235
|
+
|
|
236
|
+
```ruby
|
|
237
|
+
datastore.some.data._find("some term")
|
|
238
|
+
```
|
|
239
|
+
|
|
240
|
+
Full signature:
|
|
241
|
+
|
|
242
|
+
```ruby
|
|
243
|
+
datastore.some.data._find(term, records, tolerance)
|
|
244
|
+
```
|
|
245
|
+
|
|
246
|
+
Arguments:
|
|
247
|
+
|
|
248
|
+
- `term` - search term
|
|
249
|
+
- `records` - maximum number of records to return; defaults to `1000`
|
|
250
|
+
- `tolerance` - maximum Levenshtein distance; defaults to `50`
|
|
251
|
+
|
|
252
|
+
Results are returned as an array of entry names sorted by relevance. Entries must be within the Levenshtein tolerance and have a positive normalized bigram/Dice similarity score. The distance check is case-sensitive; similarity normalizes case, path separators, and whitespace.
|
|
253
|
+
|
|
254
|
+
Example:
|
|
255
|
+
|
|
256
|
+
```ruby
|
|
257
|
+
datastore.services._find("nginx")
|
|
258
|
+
```
|
|
259
|
+
|
|
260
|
+
might return:
|
|
261
|
+
|
|
262
|
+
```ruby
|
|
263
|
+
[
|
|
264
|
+
"nginx",
|
|
265
|
+
"nginx_config",
|
|
266
|
+
"old_nginx"
|
|
267
|
+
]
|
|
268
|
+
```
|
|
269
|
+
|
|
270
|
+
`_find` searches only the current directory level.
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
### `_exist`
|
|
274
|
+
|
|
275
|
+
Checks whether an immediate child with the specified name exists.
|
|
276
|
+
|
|
277
|
+
```ruby
|
|
278
|
+
datastore.some.data._exist("some term")
|
|
279
|
+
```
|
|
280
|
+
|
|
281
|
+
Returns:
|
|
282
|
+
|
|
283
|
+
```ruby
|
|
284
|
+
true
|
|
285
|
+
```
|
|
286
|
+
|
|
287
|
+
or:
|
|
288
|
+
|
|
289
|
+
```ruby
|
|
290
|
+
false
|
|
291
|
+
```
|
|
292
|
+
|
|
293
|
+
This method is retained for compatibility with earlier versions of OBFS.
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
### `_exists`
|
|
297
|
+
|
|
298
|
+
Checks whether the current OBFS node physically exists.
|
|
299
|
+
|
|
300
|
+
```ruby
|
|
301
|
+
datastore.some.data._exists
|
|
302
|
+
```
|
|
303
|
+
|
|
304
|
+
A question-mark alias is also available:
|
|
305
|
+
|
|
306
|
+
```ruby
|
|
307
|
+
datastore.some.data._exists?
|
|
308
|
+
```
|
|
309
|
+
|
|
310
|
+
Example:
|
|
311
|
+
|
|
312
|
+
```ruby
|
|
313
|
+
datastore.services.nginx._exists?
|
|
314
|
+
# => true
|
|
315
|
+
```
|
|
316
|
+
|
|
317
|
+
|
|
318
|
+
### `_has`
|
|
319
|
+
|
|
320
|
+
Checks whether an immediate child exists.
|
|
321
|
+
|
|
322
|
+
```ruby
|
|
323
|
+
datastore.services._has("nginx")
|
|
324
|
+
```
|
|
325
|
+
|
|
326
|
+
A question-mark alias is also available:
|
|
327
|
+
|
|
328
|
+
```ruby
|
|
329
|
+
datastore.services._has?("nginx")
|
|
330
|
+
```
|
|
331
|
+
|
|
332
|
+
Returns:
|
|
333
|
+
|
|
334
|
+
```ruby
|
|
335
|
+
true
|
|
336
|
+
```
|
|
337
|
+
|
|
338
|
+
or:
|
|
339
|
+
|
|
340
|
+
```ruby
|
|
341
|
+
false
|
|
342
|
+
```
|
|
343
|
+
|
|
344
|
+
Unlike `_exist`, `_has` performs a direct filesystem existence check rather than enumerating the current directory.
|
|
345
|
+
|
|
346
|
+
|
|
347
|
+
### `_timestamp`
|
|
348
|
+
|
|
349
|
+
Returns the modification time of the current filesystem node.
|
|
350
|
+
|
|
351
|
+
```ruby
|
|
352
|
+
datastore.services.nginx._timestamp
|
|
353
|
+
```
|
|
354
|
+
|
|
355
|
+
Returns a Ruby `Time` object when the node exists:
|
|
356
|
+
|
|
357
|
+
```ruby
|
|
358
|
+
2026-10-04 14:30:00 -0700
|
|
359
|
+
```
|
|
360
|
+
|
|
361
|
+
or:
|
|
362
|
+
|
|
363
|
+
```ruby
|
|
364
|
+
nil
|
|
365
|
+
```
|
|
366
|
+
|
|
367
|
+
if the node does not exist or cannot be read.
|
|
368
|
+
|
|
369
|
+
|
|
370
|
+
### `_search`
|
|
371
|
+
|
|
372
|
+
Performs a fuzzy search against directory names and relative directory paths.
|
|
373
|
+
|
|
374
|
+
Unlike `_find`, `_search` can recursively search the entire directory hierarchy beneath the current OBFS node.
|
|
375
|
+
|
|
376
|
+
Basic usage:
|
|
377
|
+
|
|
378
|
+
```ruby
|
|
379
|
+
datastore._search("nginx")
|
|
380
|
+
```
|
|
381
|
+
|
|
382
|
+
Example result:
|
|
383
|
+
|
|
384
|
+
```ruby
|
|
385
|
+
[
|
|
386
|
+
{
|
|
387
|
+
path: "systems/services/nginx",
|
|
388
|
+
score: 1.0
|
|
389
|
+
},
|
|
390
|
+
{
|
|
391
|
+
path: "systems/services/nginx/certificates",
|
|
392
|
+
score: 0.95
|
|
393
|
+
}
|
|
394
|
+
]
|
|
395
|
+
```
|
|
396
|
+
|
|
397
|
+
Search may also be scoped to any recursive OBFS node:
|
|
398
|
+
|
|
399
|
+
```ruby
|
|
400
|
+
datastore.systems.services._search("nginx")
|
|
401
|
+
```
|
|
402
|
+
|
|
403
|
+
Options can be supplied using a hash:
|
|
404
|
+
|
|
405
|
+
```ruby
|
|
406
|
+
datastore._search(
|
|
407
|
+
"nginx certificate",
|
|
408
|
+
{
|
|
409
|
+
recursive: true,
|
|
410
|
+
min_score: 0.6,
|
|
411
|
+
limit: 10
|
|
412
|
+
}
|
|
413
|
+
)
|
|
414
|
+
```
|
|
415
|
+
|
|
416
|
+
Available options:
|
|
417
|
+
|
|
418
|
+
- `recursive` - recursively search child directories; defaults to `true`
|
|
419
|
+
- `min_score` - minimum fuzzy relevance score from `0.0` to `1.0`; defaults to `0.5`
|
|
420
|
+
- `limit` - maximum number of results; unlimited by default
|
|
421
|
+
|
|
422
|
+
`minScore` is also accepted for compatibility with the JavaScript version:
|
|
423
|
+
|
|
424
|
+
```ruby
|
|
425
|
+
datastore._search(
|
|
426
|
+
"nginx",
|
|
427
|
+
{
|
|
428
|
+
minScore: 0.6
|
|
429
|
+
}
|
|
430
|
+
)
|
|
431
|
+
```
|
|
432
|
+
|
|
433
|
+
Search results are sorted from highest to lowest relevance.
|
|
434
|
+
|
|
435
|
+
`_search`:
|
|
436
|
+
|
|
437
|
+
- searches directory names
|
|
438
|
+
- searches relative directory paths
|
|
439
|
+
- does not read stored file contents
|
|
440
|
+
- does not follow symbolic links
|
|
441
|
+
- ignores dot-prefixed entries
|
|
442
|
+
- is scoped to the current OBFS node
|
|
443
|
+
|
|
444
|
+
|
|
445
|
+
## `_find` vs `_search`
|
|
446
|
+
|
|
447
|
+
`_find` is intended for lightweight one-level discovery:
|
|
448
|
+
|
|
449
|
+
```ruby
|
|
450
|
+
datastore.services._find("nginx")
|
|
451
|
+
```
|
|
452
|
+
|
|
453
|
+
and returns:
|
|
454
|
+
|
|
455
|
+
```ruby
|
|
456
|
+
[
|
|
457
|
+
"nginx",
|
|
458
|
+
"nginx_config"
|
|
459
|
+
]
|
|
460
|
+
```
|
|
461
|
+
|
|
462
|
+
`_search` is intended for deeper discovery:
|
|
463
|
+
|
|
464
|
+
```ruby
|
|
465
|
+
datastore._search("nginx certificate")
|
|
466
|
+
```
|
|
467
|
+
|
|
468
|
+
and returns scored relative paths:
|
|
469
|
+
|
|
470
|
+
```ruby
|
|
471
|
+
[
|
|
472
|
+
{
|
|
473
|
+
path: "systems/services/nginx/certificates",
|
|
474
|
+
score: 0.95
|
|
475
|
+
}
|
|
476
|
+
]
|
|
477
|
+
```
|
|
478
|
+
|
|
479
|
+
|
|
480
|
+
## Recursive Object Storage
|
|
481
|
+
|
|
482
|
+
One of the primary goals of OBFS is to allow the filesystem to behave like an arbitrarily recursive Ruby object.
|
|
483
|
+
|
|
484
|
+
For example:
|
|
485
|
+
|
|
486
|
+
```ruby
|
|
487
|
+
datastore.projects.example.servers.production.services.nginx.status = "running"
|
|
488
|
+
```
|
|
489
|
+
|
|
490
|
+
can later be accessed using the same structure:
|
|
491
|
+
|
|
492
|
+
```ruby
|
|
493
|
+
datastore.projects.example.servers.production.services.nginx.status
|
|
494
|
+
# => "running"
|
|
495
|
+
```
|
|
496
|
+
|
|
497
|
+
The hierarchy itself is stored using physical directories and files, making the datastore directly inspectable using normal filesystem tools.
|
|
498
|
+
|
|
499
|
+
|
|
500
|
+
## Notes
|
|
501
|
+
|
|
502
|
+
- OBFS uses the filesystem directly as its persistence layer.
|
|
503
|
+
- Intermediate paths are created automatically when writing data.
|
|
504
|
+
- Setting a value to `nil` removes the corresponding file or directory recursively.
|
|
505
|
+
- Existing values are replaced when reassigned.
|
|
506
|
+
- JSON-compatible values are serialized using Ruby's JSON library.
|
|
507
|
+
- Files containing valid JSON are deserialized when read.
|
|
508
|
+
- Non-JSON files are returned as raw text.
|
|
509
|
+
- Fuzzy search uses independent `OBFS::Levenshtein` and `OBFS::StringSimilarity` implementations.
|
|
510
|
+
- No external runtime gem dependencies are required; the Text gem is not needed.
|
|
511
|
+
- No database server is required.
|
|
512
|
+
|
|
513
|
+
|
|
514
|
+
## Platform Support
|
|
515
|
+
|
|
516
|
+
OBFS requires Ruby >= 2.0.0. The updated text algorithms use Ruby 2.0-compatible APIs. Syntax and basic behavior were validated on Ruby 4.0.7 on Linux; this update has not been tested on Ruby 2.0 itself.
|
|
517
|
+
|
|
518
|
+
Windows and macOS have not been formally tested.
|
|
519
|
+
|
|
520
|
+
|
|
521
|
+
## Links
|
|
522
|
+
|
|
523
|
+
- [RubyGems](https://rubygems.org/gems/obfs)
|
|
524
|
+
- [GitHub](https://github.com/jenselg/obfs-ruby)
|
|
525
|
+
- [JavaScript / Node.js Version](https://github.com/jenselg/obfs)
|
|
526
|
+
|
|
527
|
+
|
|
528
|
+
## Text Algorithms
|
|
529
|
+
|
|
530
|
+
The algorithms are implemented locally in `lib/obfs/levenshtein.rb` and
|
|
531
|
+
`lib/obfs/string_similarity.rb`. Requiring `obfs` loads both automatically.
|
|
532
|
+
|
|
533
|
+
### `OBFS::Levenshtein.distance`
|
|
534
|
+
|
|
535
|
+
Calculates unit-cost insertion, deletion, and substitution distance over Unicode
|
|
536
|
+
codepoints. Inputs must support String encoding; Unicode normalization is not
|
|
537
|
+
performed.
|
|
538
|
+
|
|
539
|
+
```ruby
|
|
540
|
+
OBFS::Levenshtein.distance("kitten", "sitting")
|
|
541
|
+
# => 3
|
|
542
|
+
|
|
543
|
+
OBFS::Levenshtein.distance("kitten", "sitting", 2)
|
|
544
|
+
# => 2
|
|
545
|
+
```
|
|
546
|
+
|
|
547
|
+
The optional `max_distance` must be a nonnegative Integer or `nil`. When supplied,
|
|
548
|
+
the result is capped at that maximum, preserving the existing API contract. A
|
|
549
|
+
result equal to the maximum means the actual distance is at least that value.
|
|
550
|
+
|
|
551
|
+
### `OBFS::StringSimilarity.similarity`
|
|
552
|
+
|
|
553
|
+
Returns a normalized character-bigram Dice score between `0.0` and `1.0`. It
|
|
554
|
+
normalizes case, path separators (`/`, `\`, `:`, `_`, and `-`), and whitespace.
|
|
555
|
+
Bigrams span the normalized string, including spaces, and repeated bigrams count
|
|
556
|
+
separately. Unicode codepoints are used without Unicode normalization.
|
|
557
|
+
|
|
558
|
+
```ruby
|
|
559
|
+
OBFS::StringSimilarity.similarity("France", "French")
|
|
560
|
+
# => 0.4
|
|
561
|
+
|
|
562
|
+
OBFS::StringSimilarity.similarity("nginx_config", "NGINX CONFIG")
|
|
563
|
+
# => 1.0
|
|
564
|
+
```
|
|
565
|
+
|
|
566
|
+
Equal nonempty normalized strings score `1.0`, including one-character strings.
|
|
567
|
+
Empty inputs score `0.0`; unequal one-character strings score `0.0`.
|
|
568
|
+
|
|
569
|
+
Store relevance takes the highest score from exact, prefix, substring, token
|
|
570
|
+
coverage, normalized Levenshtein, and Dice matching. `_find` and `_search` retain
|
|
571
|
+
their signatures and result formats, but rankings can change with the new Dice
|
|
572
|
+
scorer.
|
|
573
|
+
|
|
574
|
+
When upgrading, replace `lib/obfs/white_similarity.rb` with
|
|
575
|
+
`lib/obfs/string_similarity.rb` and update any direct `OBFS::WhiteSimilarity`
|
|
576
|
+
references to `OBFS::StringSimilarity`. The copied Text implementations and
|
|
577
|
+
`text/*` requires are no longer used.
|
|
578
|
+
|
|
579
|
+
|
|
580
|
+
## License
|
|
581
|
+
|
|
582
|
+
### MIT License
|
|
583
|
+
|
|
584
|
+
Copyright (c) 2021 Jensel Gatchalian <jensel.gatchalian@gmail.com>
|
|
585
|
+
|
|
586
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
587
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
588
|
+
in the Software without restriction, including without limitation the rights
|
|
589
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
590
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
591
|
+
furnished to do so, subject to the following conditions:
|
|
592
|
+
|
|
593
|
+
The above copyright notice and this permission notice shall be included in all
|
|
594
|
+
copies or substantial portions of the Software.
|
|
595
|
+
|
|
596
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
597
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
598
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
599
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
600
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
601
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
602
|
+
SOFTWARE.
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
module OBFS
|
|
2
|
+
module Levenshtein
|
|
3
|
+
extend self
|
|
4
|
+
|
|
5
|
+
# Unit-cost edit distance over Unicode codepoints, without normalization.
|
|
6
|
+
# With a maximum, return the exact distance capped at that maximum.
|
|
7
|
+
def distance(str1, str2, max_distance = nil)
|
|
8
|
+
unless max_distance.nil? ||
|
|
9
|
+
(max_distance.is_a?(Integer) && max_distance >= 0)
|
|
10
|
+
raise ArgumentError, 'max_distance must be a nonnegative Integer or nil'
|
|
11
|
+
end
|
|
12
|
+
return 0 if max_distance == 0
|
|
13
|
+
|
|
14
|
+
left = str1.encode('UTF-8').each_codepoint.to_a
|
|
15
|
+
right = str2.encode('UTF-8').each_codepoint.to_a
|
|
16
|
+
return 0 if left == right
|
|
17
|
+
|
|
18
|
+
# The shorter input determines the width of the two working rows.
|
|
19
|
+
left, right = right, left if left.length < right.length
|
|
20
|
+
if max_distance && left.length - right.length >= max_distance
|
|
21
|
+
return max_distance
|
|
22
|
+
end
|
|
23
|
+
return left.length if right.empty?
|
|
24
|
+
|
|
25
|
+
previous = (0..right.length).to_a
|
|
26
|
+
left.each_with_index do |character, row|
|
|
27
|
+
current = [row + 1]
|
|
28
|
+
right.each_with_index do |other, column|
|
|
29
|
+
current << [current[column] + 1,
|
|
30
|
+
previous[column + 1] + 1,
|
|
31
|
+
previous[column] + (character == other ? 0 : 1)].min
|
|
32
|
+
end
|
|
33
|
+
# Every path to the final cell crosses this row; costs cannot decrease.
|
|
34
|
+
return max_distance if max_distance && current.min >= max_distance
|
|
35
|
+
previous = current
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
max_distance ? [previous.last, max_distance].min : previous.last
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
end
|
data/lib/obfs/store.rb
CHANGED
|
@@ -249,7 +249,7 @@ module OBFS
|
|
|
249
249
|
|
|
250
250
|
|
|
251
251
|
similarity =
|
|
252
|
-
OBFS::
|
|
252
|
+
OBFS::StringSimilarity.similarity(
|
|
253
253
|
search_space_term,
|
|
254
254
|
term
|
|
255
255
|
)
|
|
@@ -753,8 +753,8 @@ module OBFS
|
|
|
753
753
|
# - prefix matches
|
|
754
754
|
# - substring matches
|
|
755
755
|
# - token coverage
|
|
756
|
-
# -
|
|
757
|
-
# -
|
|
756
|
+
# - OBFS Levenshtein distance
|
|
757
|
+
# - normalized bigram/Dice similarity
|
|
758
758
|
#
|
|
759
759
|
# No additional dependencies required.
|
|
760
760
|
# ------------------------------------------------------------
|
|
@@ -859,42 +859,10 @@ module OBFS
|
|
|
859
859
|
|
|
860
860
|
|
|
861
861
|
# ----------------------------------------------------------
|
|
862
|
-
#
|
|
862
|
+
# normalized bigram/Dice similarity
|
|
863
863
|
# ----------------------------------------------------------
|
|
864
864
|
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
white_score =
|
|
868
|
-
OBFS::WhiteSimilarity.similarity(
|
|
869
|
-
query,
|
|
870
|
-
value
|
|
871
|
-
).to_f
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
# support either:
|
|
875
|
-
#
|
|
876
|
-
# 0.0 - 1.0
|
|
877
|
-
#
|
|
878
|
-
# or:
|
|
879
|
-
#
|
|
880
|
-
# 0 - 100
|
|
881
|
-
#
|
|
882
|
-
|
|
883
|
-
white_score /= 100.0 if white_score > 1.0
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
white_score = 0.0 if white_score < 0.0
|
|
887
|
-
white_score = 1.0 if white_score > 1.0
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
scores << white_score
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
rescue
|
|
894
|
-
|
|
895
|
-
# Remaining scorers are sufficient.
|
|
896
|
-
|
|
897
|
-
end
|
|
865
|
+
scores << OBFS::StringSimilarity.similarity(query, value)
|
|
898
866
|
|
|
899
867
|
|
|
900
868
|
scores.empty? ? 0.0 : scores.max
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
module OBFS
|
|
2
|
+
module StringSimilarity
|
|
3
|
+
extend self
|
|
4
|
+
|
|
5
|
+
# Dice overlap of character-bigram multisets. Repeated pairs count separately.
|
|
6
|
+
# Case, path separators and whitespace are normalized before comparison.
|
|
7
|
+
# Empty text has no evidence of similarity; equal nonempty text scores 1.0.
|
|
8
|
+
def similarity(str1, str2)
|
|
9
|
+
left = normalize(str1)
|
|
10
|
+
right = normalize(str2)
|
|
11
|
+
return 0.0 if left.empty? || right.empty?
|
|
12
|
+
return 1.0 if left == right
|
|
13
|
+
|
|
14
|
+
left_pairs = bigrams(left)
|
|
15
|
+
right_pairs = bigrams(right)
|
|
16
|
+
total = left_pairs.values.inject(0, :+) + right_pairs.values.inject(0, :+)
|
|
17
|
+
return 0.0 if total == 0
|
|
18
|
+
|
|
19
|
+
overlap = left_pairs.inject(0) do |sum, entry|
|
|
20
|
+
pair, count = entry
|
|
21
|
+
sum + [count, right_pairs[pair]].min
|
|
22
|
+
end
|
|
23
|
+
2.0 * overlap / total
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
private
|
|
27
|
+
|
|
28
|
+
def normalize(value)
|
|
29
|
+
value.to_s.encode('UTF-8').downcase
|
|
30
|
+
.gsub(/[\\\/:_\-]+/, ' ')
|
|
31
|
+
.gsub(/\s+/, ' ').strip
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def bigrams(value)
|
|
35
|
+
counts = Hash.new(0)
|
|
36
|
+
characters = value.each_codepoint.to_a
|
|
37
|
+
characters.each_cons(2) { |pair| counts[pair] += 1 }
|
|
38
|
+
counts
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
end
|
data/lib/obfs.rb
CHANGED
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: obfs
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.3.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Jensel Gatchalian
|
|
@@ -9,16 +9,18 @@ bindir: bin
|
|
|
9
9
|
cert_chain: []
|
|
10
10
|
date: 2026-10-04 00:00:00.000000000 Z
|
|
11
11
|
dependencies: []
|
|
12
|
-
description: File-based, object-oriented data store for Ruby
|
|
12
|
+
description: File-based, object-oriented data store for Ruby with recursive access,
|
|
13
|
+
fuzzy discovery, and no external runtime dependencies.
|
|
13
14
|
email: jensel.gatchalian@gmail.com
|
|
14
15
|
executables: []
|
|
15
16
|
extensions: []
|
|
16
17
|
extra_rdoc_files: []
|
|
17
18
|
files:
|
|
19
|
+
- README.md
|
|
18
20
|
- lib/obfs.rb
|
|
21
|
+
- lib/obfs/levenshtein.rb
|
|
19
22
|
- lib/obfs/store.rb
|
|
20
|
-
- lib/
|
|
21
|
-
- lib/text/white_similarity.rb
|
|
23
|
+
- lib/obfs/string_similarity.rb
|
|
22
24
|
homepage: https://github.com/jenselg/obfs-ruby
|
|
23
25
|
licenses:
|
|
24
26
|
- MIT
|
|
@@ -39,5 +41,5 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
39
41
|
requirements: []
|
|
40
42
|
rubygems_version: 4.0.20
|
|
41
43
|
specification_version: 4
|
|
42
|
-
summary:
|
|
44
|
+
summary: Filesystem-backed object store with fuzzy discovery
|
|
43
45
|
test_files: []
|
data/lib/text/levenshtein.rb
DELETED
|
@@ -1,166 +0,0 @@
|
|
|
1
|
-
#
|
|
2
|
-
# Credits to threedaymonk
|
|
3
|
-
# https://github.com/threedaymonk/text/blob/master/lib/text/levenshtein.rb
|
|
4
|
-
#
|
|
5
|
-
# Levenshtein distance algorithm implementation for Ruby, with UTF-8 support.
|
|
6
|
-
#
|
|
7
|
-
# The Levenshtein distance is a measure of how similar two strings s and t are,
|
|
8
|
-
# calculated as the number of deletions/insertions/substitutions needed to
|
|
9
|
-
# transform s into t. The greater the distance, the more the strings differ.
|
|
10
|
-
#
|
|
11
|
-
# The Levenshtein distance is also sometimes referred to as the
|
|
12
|
-
# easier-to-pronounce-and-spell 'edit distance'.
|
|
13
|
-
#
|
|
14
|
-
# Author: Paul Battley (pbattley@gmail.com)
|
|
15
|
-
#
|
|
16
|
-
|
|
17
|
-
module OBFS # :nodoc:
|
|
18
|
-
|
|
19
|
-
module Levenshtein
|
|
20
|
-
|
|
21
|
-
# Calculate the Levenshtein distance between two strings +str1+ and +str2+.
|
|
22
|
-
#
|
|
23
|
-
# The optional argument max_distance can reduce the number of iterations by
|
|
24
|
-
# stopping if the Levenshtein distance exceeds this value. This increases
|
|
25
|
-
# performance where it is only necessary to compare the distance with a
|
|
26
|
-
# reference value instead of calculating the exact distance.
|
|
27
|
-
#
|
|
28
|
-
# The distance is calculated in terms of Unicode codepoints. Be aware that
|
|
29
|
-
# this algorithm does not perform normalisation: if there is a possibility
|
|
30
|
-
# of different normalised forms being used, normalisation should be performed
|
|
31
|
-
# beforehand.
|
|
32
|
-
#
|
|
33
|
-
def distance(str1, str2, max_distance = nil)
|
|
34
|
-
if max_distance
|
|
35
|
-
distance_with_maximum(str1, str2, max_distance)
|
|
36
|
-
else
|
|
37
|
-
distance_without_maximum(str1, str2)
|
|
38
|
-
end
|
|
39
|
-
end
|
|
40
|
-
|
|
41
|
-
private
|
|
42
|
-
|
|
43
|
-
def distance_with_maximum(str1, str2, max_distance) # :nodoc:
|
|
44
|
-
s = str1.encode(Encoding::UTF_8).unpack("U*")
|
|
45
|
-
t = str2.encode(Encoding::UTF_8).unpack("U*")
|
|
46
|
-
|
|
47
|
-
n = s.length
|
|
48
|
-
m = t.length
|
|
49
|
-
big_int = n * m
|
|
50
|
-
|
|
51
|
-
# Swap if necessary so that s is always the shorter of the two strings
|
|
52
|
-
s, t, n, m = t, s, m, n if m < n
|
|
53
|
-
|
|
54
|
-
# If the length difference is already greater than the max_distance, then
|
|
55
|
-
# there is nothing else to check
|
|
56
|
-
if (n - m).abs >= max_distance
|
|
57
|
-
return max_distance
|
|
58
|
-
end
|
|
59
|
-
|
|
60
|
-
return 0 if s == t
|
|
61
|
-
return m if n.zero?
|
|
62
|
-
return n if m.zero?
|
|
63
|
-
|
|
64
|
-
# The values necessary for our threshold are written; the ones after must
|
|
65
|
-
# be filled with large integers since the tailing member of the threshold
|
|
66
|
-
# window in the bottom array will run min across them
|
|
67
|
-
d = (m + 1).times.map { |i|
|
|
68
|
-
if i < m || i < max_distance + 1
|
|
69
|
-
i
|
|
70
|
-
else
|
|
71
|
-
big_int
|
|
72
|
-
end
|
|
73
|
-
}
|
|
74
|
-
x = nil
|
|
75
|
-
e = nil
|
|
76
|
-
|
|
77
|
-
n.times do |i|
|
|
78
|
-
# Since we're reusing arrays, we need to be sure to wipe the value left
|
|
79
|
-
# of the starting index; we don't have to worry about the value above the
|
|
80
|
-
# ending index as the arrays were initially filled with large integers
|
|
81
|
-
# and we progress to the right
|
|
82
|
-
if e.nil?
|
|
83
|
-
e = i + 1
|
|
84
|
-
else
|
|
85
|
-
e = big_int
|
|
86
|
-
end
|
|
87
|
-
|
|
88
|
-
diag_index = t.length - s.length + i
|
|
89
|
-
|
|
90
|
-
# If max_distance was specified, we can reduce second loop. So we set
|
|
91
|
-
# up our threshold window.
|
|
92
|
-
# See:
|
|
93
|
-
# Gusfield, Dan (1997). Algorithms on strings, trees, and sequences:
|
|
94
|
-
# computer science and computational biology.
|
|
95
|
-
# Cambridge, UK: Cambridge University Press. ISBN 0-521-58519-8.
|
|
96
|
-
# pp. 263–264.
|
|
97
|
-
min = i - max_distance - 1
|
|
98
|
-
min = 0 if min < 0
|
|
99
|
-
max = i + max_distance
|
|
100
|
-
max = m - 1 if max > m - 1
|
|
101
|
-
|
|
102
|
-
min.upto(max) do |j|
|
|
103
|
-
# If the diagonal value is already greater than the max_distance
|
|
104
|
-
# then we can safety return: the diagonal will never go lower again.
|
|
105
|
-
# See: http://www.levenshtein.net/
|
|
106
|
-
if j == diag_index && d[j] >= max_distance
|
|
107
|
-
return max_distance
|
|
108
|
-
end
|
|
109
|
-
|
|
110
|
-
cost = s[i] == t[j] ? 0 : 1
|
|
111
|
-
insertion = d[j + 1] + 1
|
|
112
|
-
deletion = e + 1
|
|
113
|
-
substitution = d[j] + cost
|
|
114
|
-
x = insertion < deletion ? insertion : deletion
|
|
115
|
-
x = substitution if substitution < x
|
|
116
|
-
|
|
117
|
-
d[j] = e
|
|
118
|
-
e = x
|
|
119
|
-
end
|
|
120
|
-
d[m] = x
|
|
121
|
-
end
|
|
122
|
-
|
|
123
|
-
if x > max_distance
|
|
124
|
-
return max_distance
|
|
125
|
-
else
|
|
126
|
-
return x
|
|
127
|
-
end
|
|
128
|
-
end
|
|
129
|
-
|
|
130
|
-
def distance_without_maximum(str1, str2) # :nodoc:
|
|
131
|
-
s = str1.encode(Encoding::UTF_8).unpack("U*")
|
|
132
|
-
t = str2.encode(Encoding::UTF_8).unpack("U*")
|
|
133
|
-
|
|
134
|
-
n = s.length
|
|
135
|
-
m = t.length
|
|
136
|
-
|
|
137
|
-
return m if n.zero?
|
|
138
|
-
return n if m.zero?
|
|
139
|
-
|
|
140
|
-
d = (0..m).to_a
|
|
141
|
-
x = nil
|
|
142
|
-
|
|
143
|
-
n.times do |i|
|
|
144
|
-
e = i + 1
|
|
145
|
-
m.times do |j|
|
|
146
|
-
cost = s[i] == t[j] ? 0 : 1
|
|
147
|
-
insertion = d[j + 1] + 1
|
|
148
|
-
deletion = e + 1
|
|
149
|
-
substitution = d[j] + cost
|
|
150
|
-
x = insertion < deletion ? insertion : deletion
|
|
151
|
-
x = substitution if substitution < x
|
|
152
|
-
|
|
153
|
-
d[j] = e
|
|
154
|
-
e = x
|
|
155
|
-
end
|
|
156
|
-
d[m] = x
|
|
157
|
-
end
|
|
158
|
-
|
|
159
|
-
return x
|
|
160
|
-
end
|
|
161
|
-
|
|
162
|
-
extend self
|
|
163
|
-
|
|
164
|
-
end
|
|
165
|
-
|
|
166
|
-
end
|
|
@@ -1,66 +0,0 @@
|
|
|
1
|
-
#
|
|
2
|
-
# Credits to threedaymonk
|
|
3
|
-
# https://github.com/threedaymonk/text/blob/master/lib/text/white_similarity.rb
|
|
4
|
-
#
|
|
5
|
-
# encoding: utf-8
|
|
6
|
-
# Original author: Wilker Lúcio <wilkerlucio@gmail.com>
|
|
7
|
-
|
|
8
|
-
module OBFS
|
|
9
|
-
|
|
10
|
-
# Ruby implementation of the string similarity described by Simon White
|
|
11
|
-
# at: http://www.catalysoft.com/articles/StrikeAMatch.html
|
|
12
|
-
#
|
|
13
|
-
# 2 * |pairs(s1) INTERSECT pairs(s2)|
|
|
14
|
-
# similarity(s1, s2) = -----------------------------------
|
|
15
|
-
# |pairs(s1)| + |pairs(s2)|
|
|
16
|
-
#
|
|
17
|
-
# e.g.
|
|
18
|
-
# 2 * |{FR, NC}|
|
|
19
|
-
# similarity(FRANCE, FRENCH) = ---------------------------------------
|
|
20
|
-
# |{FR,RA,AN,NC,CE}| + |{FR,RE,EN,NC,CH}|
|
|
21
|
-
#
|
|
22
|
-
# = (2 * 2) / (5 + 5)
|
|
23
|
-
#
|
|
24
|
-
# = 0.4
|
|
25
|
-
#
|
|
26
|
-
# WhiteSimilarity.new.similarity("FRANCE", "FRENCH")
|
|
27
|
-
#
|
|
28
|
-
class WhiteSimilarity
|
|
29
|
-
|
|
30
|
-
def self.similarity(str1, str2)
|
|
31
|
-
new.similarity(str1, str2)
|
|
32
|
-
end
|
|
33
|
-
|
|
34
|
-
def initialize
|
|
35
|
-
@word_letter_pairs = {}
|
|
36
|
-
end
|
|
37
|
-
|
|
38
|
-
def similarity(str1, str2)
|
|
39
|
-
pairs1 = word_letter_pairs(str1)
|
|
40
|
-
pairs2 = word_letter_pairs(str2).dup
|
|
41
|
-
|
|
42
|
-
union = pairs1.length + pairs2.length
|
|
43
|
-
|
|
44
|
-
intersection = 0
|
|
45
|
-
pairs1.each do |pair1|
|
|
46
|
-
if index = pairs2.index(pair1)
|
|
47
|
-
intersection += 1
|
|
48
|
-
pairs2.delete_at(index)
|
|
49
|
-
end
|
|
50
|
-
end
|
|
51
|
-
|
|
52
|
-
(2.0 * intersection) / union
|
|
53
|
-
end
|
|
54
|
-
|
|
55
|
-
private
|
|
56
|
-
|
|
57
|
-
def word_letter_pairs(str)
|
|
58
|
-
@word_letter_pairs[str] ||=
|
|
59
|
-
str.upcase.split(/\s+/).map{ |word|
|
|
60
|
-
(0 ... (word.length - 1)).map { |i| word[i, 2] }
|
|
61
|
-
}.flatten.freeze
|
|
62
|
-
end
|
|
63
|
-
|
|
64
|
-
end
|
|
65
|
-
|
|
66
|
-
end
|