unisat 0.0.1 → 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +1 -0
- data/README.md +82 -5
- data/Rakefile +1 -1
- data/lib/unisat/cache.rb +45 -28
- data/lib/unisat/version.rb +2 -2
- data/lib/unisat.rb +54 -27
- metadata +4 -5
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 757089e74852538e2d5803dd6995ea61dbc8f709b1b321cdec45d985b82d2562
|
|
4
|
+
data.tar.gz: dab383d468337d59a928f64cbb3a831c5e6a09deed6e18c468c97204b74e5a78
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 4b5270ac0575703df5aa59a0eae060b29e3ff8c7cabf03daa6ba90be839e8ab673decc09a2fcb526e7490cf98b48e85caaa85cc61e419e0bf0e318dfc89d2fec
|
|
7
|
+
data.tar.gz: a8b1596b0b1a7fa31501d218a9640cd5ef958b4a7d901d8d14dee64137dc0e0073b0d4f151de92947a075fa97dbf446864ccfdd4cf77aee40ff0422ee307eff1
|
data/CHANGELOG.md
CHANGED
data/README.md
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# UniSat.io Lite Helpers for Full-Text Search in Bitcoin Ordinal Inscriptions
|
|
2
2
|
|
|
3
|
-
unisat - lite helpers for full-text search in bitcoin ordinal inscriptions
|
|
3
|
+
unisat - unisat.io lite helpers for full-text search in bitcoin ordinal inscriptions
|
|
4
|
+
|
|
4
5
|
|
|
5
6
|
* home :: [github.com/ordbase/unisat](https://github.com/ordbase/unisat)
|
|
6
7
|
* bugs :: [github.com/ordbase/unisat/issues](https://github.com/ordbase/unisat/issues)
|
|
@@ -11,10 +12,86 @@ unisat - lite helpers for full-text search in bitcoin ordinal inscriptions power
|
|
|
11
12
|
|
|
12
13
|
## Usage
|
|
13
14
|
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
15
|
+
Note: If you full-text search on UniSat.io by trying to fetch a "static" web page
|
|
16
|
+
with a script you will NOT the get the results BUT end-up with an "empty-shell"
|
|
17
|
+
because the inscriptions get fetched "delayed" via js/json.
|
|
18
|
+
To work-around use puppeteer to automate the google chrome browser with your scripts.
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
Let's full-text search in bitcoin ordinal inscriptions for "diyphunks".
|
|
22
|
+
|
|
23
|
+
```ruby
|
|
24
|
+
require 'unisat'
|
|
25
|
+
|
|
26
|
+
## Step 0: Configure the file path to your local google chrome browser
|
|
27
|
+
## required by puppeeter.
|
|
28
|
+
Unisat::Puppeteer.chrome_path = "your/path/to/chrome/here"
|
|
29
|
+
|
|
30
|
+
## Step 1:
|
|
31
|
+
recs = Unisat::Puppeteer.search( 'diyphunks' )
|
|
32
|
+
puts " #{recs.size} inscribe(s)"
|
|
33
|
+
#=> 21 inscribe(s)
|
|
34
|
+
|
|
35
|
+
pp recs
|
|
36
|
+
#=>[{"id"=>"09e3c57879b0a30239b088468277d009f838111d254eb22f35a9b2a31b77a5d4i0",
|
|
37
|
+
# "num"=>"13418851",
|
|
38
|
+
# "date"=>"24.6.2023, 22:41:33",
|
|
39
|
+
# "address"=>"bc1pmgu...quyqrah",
|
|
40
|
+
# "text"=> "og deploy diyphunks↵name: D.I.Y. Phunks↵max: 100↵dim: ..."},
|
|
41
|
+
# {"id"=>"54a448aad17203e322b3d421d91c12e1e4c1b1ca46ef2849e52774aeb7a3bd3bi0",
|
|
42
|
+
# "num"=>"13418907",
|
|
43
|
+
# "date"=>"24.6.2023, 22:45:20",
|
|
44
|
+
# "address"=>"bc1pmgu...quyqrah",
|
|
45
|
+
# "text"=>"og mint diyphunks 5 57 38"},
|
|
46
|
+
# ...]
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
That's it.
|
|
50
|
+
|
|
51
|
+
Let's full-text search in bitcoin ordinal inscriptions for "orangepixels"
|
|
52
|
+
to try the "magic" pagination
|
|
53
|
+
auto-adding pages with 32 results each.
|
|
54
|
+
|
|
55
|
+
Note: If you want to use your own page (starting) offset (default is 1)
|
|
56
|
+
and limit (default is none),
|
|
57
|
+
use the optional `òffset` and `limit` keyword parameters.
|
|
58
|
+
|
|
59
|
+
``` ruby
|
|
60
|
+
recs = Unisat::Puppeteer.search( 'orangepixels' )
|
|
61
|
+
puts " #{recs.size} inscribe(s)"
|
|
62
|
+
#=> 101 inscribe(s)
|
|
63
|
+
|
|
64
|
+
pp recs
|
|
65
|
+
#=> [{"id"=>"429915c362dacc2b1f7d4a5f7d929ee04e52298c35a7fe9c4e5a29580691b364i0",
|
|
66
|
+
# "num"=>"13070954",
|
|
67
|
+
# "date"=>"21.6.2023, 20:15:12",
|
|
68
|
+
# "address"=>"bc1qjhu...5lla3hj",
|
|
69
|
+
# "text"=>"{↵ \"p\": \"orc-721\",↵ \"op\": \"deploy\",..."},
|
|
70
|
+
# {"id"=>"157fdb5ebe82c12cefb6d48356d79bf5646e267980a2234294a8a330a4234be7i0",
|
|
71
|
+
# "num"=>"13074990",
|
|
72
|
+
# "date"=>"21.6.2023, 21:18:02",
|
|
73
|
+
# "address"=>"bc1qjhu...5lla3hj",
|
|
74
|
+
# "text"=>"{↵ \"p\":\"orc-721\",↵ \"op\":\"mint\",..."},
|
|
75
|
+
# ...]
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
Note: For now all results get "automagically" cached in `unisat/` indexd
|
|
80
|
+
by query (key) and pages. Example:
|
|
81
|
+
|
|
82
|
+
```
|
|
83
|
+
/unisat
|
|
84
|
+
/diyphunks
|
|
85
|
+
1.json
|
|
86
|
+
/orangepixels
|
|
87
|
+
1.json
|
|
88
|
+
2.json
|
|
89
|
+
3.json
|
|
90
|
+
4.json
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
That's it.
|
|
18
95
|
|
|
19
96
|
|
|
20
97
|
|
data/Rakefile
CHANGED
|
@@ -5,7 +5,7 @@ Hoe.spec 'unisat' do
|
|
|
5
5
|
|
|
6
6
|
self.version = Unisat::VERSION
|
|
7
7
|
|
|
8
|
-
self.summary = "unisat - lite helpers for full-text search in bitcoin ordinal inscriptions
|
|
8
|
+
self.summary = "unisat - unisat.io lite helpers for full-text search in bitcoin ordinal inscriptions"
|
|
9
9
|
self.description = summary
|
|
10
10
|
|
|
11
11
|
self.urls = { home: 'https://github.com/ordbase/ordbase' }
|
data/lib/unisat/cache.rb
CHANGED
|
@@ -40,6 +40,7 @@ def self.parse_page( html )
|
|
|
40
40
|
end # method _parse_page
|
|
41
41
|
|
|
42
42
|
|
|
43
|
+
|
|
43
44
|
class Cache
|
|
44
45
|
def initialize( dir )
|
|
45
46
|
@dir = dir
|
|
@@ -57,52 +58,68 @@ class Cache
|
|
|
57
58
|
results = read_json( path )
|
|
58
59
|
puts "==> #{i+1}/#{paths.size} - #{results.size} inscribe(s) in >#{path}<..."
|
|
59
60
|
|
|
60
|
-
results
|
|
61
|
-
date = h['date'] # note: might include unconfirmed!!!
|
|
62
|
-
## always "auto-magically" filter out unconfirmed for now
|
|
63
|
-
if date == 'unconfirmed'
|
|
64
|
-
puts " !! INFO - skipping unconfirmed inscribe"
|
|
65
|
-
next
|
|
66
|
-
end
|
|
67
|
-
|
|
68
|
-
## todo/fix:
|
|
69
|
-
## reformat date here/parse
|
|
70
|
-
## and change to iso e.g. 2023-12-24 17:18 or such - why? why not!!!!
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
## id -- split href by / and get last part (is id)
|
|
74
|
-
id = h['href'].split( '/')[-1]
|
|
75
|
-
## num -- remove/ strip leadingnumber sign (#)
|
|
76
|
-
num = h['num'].sub( '#', '' )
|
|
77
|
-
recs << {
|
|
78
|
-
'id' => id,
|
|
79
|
-
'num' => num,
|
|
80
|
-
'date' => date,
|
|
81
|
-
'address' => h['address'],
|
|
82
|
-
'text' => h['name'], ## change name to text
|
|
83
|
-
}
|
|
84
|
-
end
|
|
61
|
+
recs += _normalize( results )
|
|
85
62
|
end
|
|
86
63
|
puts " #{recs.size} inscribe(s) - total"
|
|
87
64
|
recs
|
|
88
65
|
end # method read
|
|
89
66
|
|
|
90
67
|
|
|
68
|
+
def _normalize( results )
|
|
69
|
+
recs = []
|
|
70
|
+
results.each do |h|
|
|
71
|
+
date = h['date'] # note: might include unconfirmed!!!
|
|
72
|
+
## always "auto-magically" filter out unconfirmed for now
|
|
73
|
+
if date == 'unconfirmed'
|
|
74
|
+
puts " !! INFO - skipping unconfirmed inscribe"
|
|
75
|
+
next
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
## todo/fix:
|
|
79
|
+
## reformat date here/parse
|
|
80
|
+
## and change to iso e.g. 2023-12-24 17:18 or such - why? why not!!!!
|
|
81
|
+
|
|
82
|
+
## id -- split href by / and get last part (is id)
|
|
83
|
+
id = h['href'].split( '/')[-1]
|
|
84
|
+
## num -- remove/ strip leadingnumber sign (#)
|
|
85
|
+
num = h['num'].sub( '#', '' )
|
|
86
|
+
recs << {
|
|
87
|
+
'id' => id,
|
|
88
|
+
'num' => num,
|
|
89
|
+
'date' => date,
|
|
90
|
+
'address' => h['address'],
|
|
91
|
+
'text' => h['name'], ## change name to text
|
|
92
|
+
}
|
|
93
|
+
end
|
|
94
|
+
recs
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
|
|
91
98
|
def exist?( key, offset: )
|
|
92
99
|
outpath = "#{@dir}/#{key}/#{offset}.json"
|
|
93
100
|
File.exist?( outpath )
|
|
94
101
|
end
|
|
95
102
|
|
|
103
|
+
def read_page( key, offset: )
|
|
104
|
+
outpath = "#{@dir}/#{key}/#{offset}.json"
|
|
105
|
+
results = read_json( outpath )
|
|
106
|
+
recs = _normalize( results )
|
|
107
|
+
recs
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
|
|
96
111
|
def add_page( page, key, offset: )
|
|
97
|
-
|
|
112
|
+
results = Unisat.parse_page( page )
|
|
98
113
|
|
|
99
114
|
## note: only write if results > 0
|
|
100
|
-
if
|
|
115
|
+
if results.size > 0
|
|
101
116
|
outpath = "#{@dir}/#{key}/#{offset}.json"
|
|
102
|
-
write_json( outpath,
|
|
117
|
+
write_json( outpath, results )
|
|
103
118
|
else
|
|
104
119
|
puts "!! WARN - no results found in page #{offset} for key >#{key}<"
|
|
105
120
|
end
|
|
121
|
+
recs = _normalize( results )
|
|
122
|
+
recs
|
|
106
123
|
end
|
|
107
124
|
end # class Cache
|
|
108
125
|
end # module Unisat
|
data/lib/unisat/version.rb
CHANGED
data/lib/unisat.rb
CHANGED
|
@@ -40,30 +40,48 @@ def self.cache() Unisat.cache; end
|
|
|
40
40
|
###
|
|
41
41
|
# search
|
|
42
42
|
def self.search( q, offset: 1,
|
|
43
|
-
|
|
44
|
-
|
|
43
|
+
force: false,
|
|
44
|
+
limit: nil )
|
|
45
45
|
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
46
|
+
browser = nil
|
|
47
|
+
page = nil
|
|
48
|
+
recs = []
|
|
49
49
|
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
50
|
+
print "==> searching >#{q}< starting at page #{offset}"
|
|
51
|
+
print " up to #{offset+limit-1} (#{limit} pages)" if limit && limit > 1
|
|
52
|
+
print ", force: true" if force
|
|
53
|
+
print "...\n"
|
|
54
54
|
|
|
55
|
-
|
|
55
|
+
page_recs = nil
|
|
56
|
+
i = 0
|
|
57
|
+
loop do
|
|
58
|
+
break if limit && i >= limit
|
|
56
59
|
count = offset+i
|
|
57
60
|
|
|
58
61
|
page_url = "https://unisat.io/search?q=#{q}&type=text&p=#{count}"
|
|
59
62
|
print page_url
|
|
60
63
|
|
|
61
64
|
## check if already in cache
|
|
62
|
-
if force == false && cache.exist?( q,
|
|
65
|
+
if force == false && cache.exist?( q,
|
|
66
|
+
offset: count )
|
|
63
67
|
print "...in cache\n"
|
|
68
|
+
page_recs = cache.read_page( q, offset: count )
|
|
69
|
+
recs += page_recs
|
|
70
|
+
|
|
71
|
+
break if limit.nil? && page_recs.size < 32 ## check for auto-limit (if less than 32 recs)
|
|
72
|
+
i += 1
|
|
64
73
|
next
|
|
65
74
|
end
|
|
66
75
|
|
|
76
|
+
if browser.nil? ## first page request? launch browser on demand
|
|
77
|
+
opts = {}
|
|
78
|
+
opts[:headless] = false
|
|
79
|
+
opts[:executable_path] = chrome_path if chrome_path ## add only if set (default is nil)
|
|
80
|
+
|
|
81
|
+
browser = ::Puppeteer.launch( **opts )
|
|
82
|
+
page = browser.new_page
|
|
83
|
+
end
|
|
84
|
+
|
|
67
85
|
print "... goto page ...\n"
|
|
68
86
|
response = page.goto( page_url )
|
|
69
87
|
pp response.headers
|
|
@@ -77,16 +95,7 @@ def self.search( q, offset: 1,
|
|
|
77
95
|
puts
|
|
78
96
|
puts el.evaluate("el => el.innerText")
|
|
79
97
|
#=> Searched for "biixel" among 3252336 records, found 1105 results.
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
## print pagination
|
|
83
|
-
page.wait_for_selector('ul.ant-pagination')
|
|
84
|
-
el = page.query_selector("ul.ant-pagination")
|
|
85
|
-
puts
|
|
86
|
-
puts el.evaluate("el => el.innerText")
|
|
87
|
-
#=> 1 ••• 32 33 34 35
|
|
88
|
-
|
|
89
|
-
|
|
98
|
+
|
|
90
99
|
## get search results (32 inscribes per page)
|
|
91
100
|
page.wait_for_selector('div.sats-list')
|
|
92
101
|
el = page.query_selector("div.sats-list")
|
|
@@ -94,14 +103,32 @@ def self.search( q, offset: 1,
|
|
|
94
103
|
html = el.evaluate("el => el.innerHTML")
|
|
95
104
|
# puts
|
|
96
105
|
# puts html
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
106
|
+
|
|
107
|
+
## print pagination
|
|
108
|
+
## note: only available if more than one page!!!
|
|
109
|
+
el = page.query_selector("ul.ant-pagination")
|
|
110
|
+
if el
|
|
111
|
+
puts
|
|
112
|
+
puts el.evaluate("el => el.innerText")
|
|
113
|
+
#=> 1 ••• 32 33 34 35
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
page_recs = cache.add_page( html, q,
|
|
117
|
+
offset: count )
|
|
118
|
+
recs += page_recs
|
|
119
|
+
|
|
120
|
+
break if limit.nil? && page_recs.size < 32 ## check for auto-limit (if less than 32 recs)
|
|
121
|
+
|
|
122
|
+
i += 1
|
|
123
|
+
end # loop
|
|
124
|
+
|
|
125
|
+
if browser
|
|
126
|
+
puts "sleeping 2 secs before browser shutdown..."
|
|
127
|
+
sleep( 2 )
|
|
128
|
+
browser.close
|
|
100
129
|
end
|
|
101
130
|
|
|
102
|
-
|
|
103
|
-
sleep( 2 )
|
|
104
|
-
end
|
|
131
|
+
recs
|
|
105
132
|
end # method search
|
|
106
133
|
end # module Puppeteer
|
|
107
134
|
end # module Unisat
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: unisat
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.0
|
|
4
|
+
version: 0.1.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Gerald Bauer
|
|
@@ -86,8 +86,8 @@ dependencies:
|
|
|
86
86
|
- - "~>"
|
|
87
87
|
- !ruby/object:Gem::Version
|
|
88
88
|
version: '4.0'
|
|
89
|
-
description: unisat - lite helpers for full-text search in bitcoin ordinal
|
|
90
|
-
|
|
89
|
+
description: unisat - unisat.io lite helpers for full-text search in bitcoin ordinal
|
|
90
|
+
inscriptions
|
|
91
91
|
email: gerald.bauer@gmail.com
|
|
92
92
|
executables: []
|
|
93
93
|
extensions: []
|
|
@@ -127,6 +127,5 @@ requirements: []
|
|
|
127
127
|
rubygems_version: 3.4.10
|
|
128
128
|
signing_key:
|
|
129
129
|
specification_version: 4
|
|
130
|
-
summary: unisat - lite helpers for full-text search in bitcoin ordinal inscriptions
|
|
131
|
-
powered by unisat.io
|
|
130
|
+
summary: unisat - unisat.io lite helpers for full-text search in bitcoin ordinal inscriptions
|
|
132
131
|
test_files: []
|