unisat 0.0.1 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +1 -0
- data/README.md +82 -5
- data/Rakefile +1 -1
- data/lib/unisat/cache.rb +45 -28
- data/lib/unisat/version.rb +2 -2
- data/lib/unisat.rb +54 -27
- metadata +4 -5
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 757089e74852538e2d5803dd6995ea61dbc8f709b1b321cdec45d985b82d2562
|
|
4
|
+
data.tar.gz: dab383d468337d59a928f64cbb3a831c5e6a09deed6e18c468c97204b74e5a78
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 4b5270ac0575703df5aa59a0eae060b29e3ff8c7cabf03daa6ba90be839e8ab673decc09a2fcb526e7490cf98b48e85caaa85cc61e419e0bf0e318dfc89d2fec
|
|
7
|
+
data.tar.gz: a8b1596b0b1a7fa31501d218a9640cd5ef958b4a7d901d8d14dee64137dc0e0073b0d4f151de92947a075fa97dbf446864ccfdd4cf77aee40ff0422ee307eff1
|
data/CHANGELOG.md
CHANGED
data/README.md
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# UniSat.io Lite Helpers for Full-Text Search in Bitcoin Ordinal Inscriptions
|
|
2
2
|
|
|
3
|
-
unisat - lite helpers for full-text search in bitcoin ordinal inscriptions
|
|
3
|
+
unisat - unisat.io lite helpers for full-text search in bitcoin ordinal inscriptions
|
|
4
|
+
|
|
4
5
|
|
|
5
6
|
* home :: [github.com/ordbase/unisat](https://github.com/ordbase/unisat)
|
|
6
7
|
* bugs :: [github.com/ordbase/unisat/issues](https://github.com/ordbase/unisat/issues)
|
|
@@ -11,10 +12,86 @@ unisat - lite helpers for full-text search in bitcoin ordinal inscriptions power
|
|
|
11
12
|
|
|
12
13
|
## Usage
|
|
13
14
|
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
15
|
+
Note: If you full-text search on UniSat.io by trying to fetch a "static" web page
|
|
16
|
+
with a script you will NOT the get the results BUT end-up with an "empty-shell"
|
|
17
|
+
because the inscriptions get fetched "delayed" via js/json.
|
|
18
|
+
To work-around use puppeteer to automate the google chrome browser with your scripts.
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
Let's full-text search in bitcoin ordinal inscriptions for "diyphunks".
|
|
22
|
+
|
|
23
|
+
```ruby
|
|
24
|
+
require 'unisat'
|
|
25
|
+
|
|
26
|
+
## Step 0: Configure the file path to your local google chrome browser
|
|
27
|
+
## required by puppeeter.
|
|
28
|
+
Unisat::Puppeteer.chrome_path = "your/path/to/chrome/here"
|
|
29
|
+
|
|
30
|
+
## Step 1:
|
|
31
|
+
recs = Unisat::Puppeteer.search( 'diyphunks' )
|
|
32
|
+
puts " #{recs.size} inscribe(s)"
|
|
33
|
+
#=> 21 inscribe(s)
|
|
34
|
+
|
|
35
|
+
pp recs
|
|
36
|
+
#=>[{"id"=>"09e3c57879b0a30239b088468277d009f838111d254eb22f35a9b2a31b77a5d4i0",
|
|
37
|
+
# "num"=>"13418851",
|
|
38
|
+
# "date"=>"24.6.2023, 22:41:33",
|
|
39
|
+
# "address"=>"bc1pmgu...quyqrah",
|
|
40
|
+
# "text"=> "og deploy diyphunks↵name: D.I.Y. Phunks↵max: 100↵dim: ..."},
|
|
41
|
+
# {"id"=>"54a448aad17203e322b3d421d91c12e1e4c1b1ca46ef2849e52774aeb7a3bd3bi0",
|
|
42
|
+
# "num"=>"13418907",
|
|
43
|
+
# "date"=>"24.6.2023, 22:45:20",
|
|
44
|
+
# "address"=>"bc1pmgu...quyqrah",
|
|
45
|
+
# "text"=>"og mint diyphunks 5 57 38"},
|
|
46
|
+
# ...]
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
That's it.
|
|
50
|
+
|
|
51
|
+
Let's full-text search in bitcoin ordinal inscriptions for "orangepixels"
|
|
52
|
+
to try the "magic" pagination
|
|
53
|
+
auto-adding pages with 32 results each.
|
|
54
|
+
|
|
55
|
+
Note: If you want to use your own page (starting) offset (default is 1)
|
|
56
|
+
and limit (default is none),
|
|
57
|
+
use the optional `òffset` and `limit` keyword parameters.
|
|
58
|
+
|
|
59
|
+
``` ruby
|
|
60
|
+
recs = Unisat::Puppeteer.search( 'orangepixels' )
|
|
61
|
+
puts " #{recs.size} inscribe(s)"
|
|
62
|
+
#=> 101 inscribe(s)
|
|
63
|
+
|
|
64
|
+
pp recs
|
|
65
|
+
#=> [{"id"=>"429915c362dacc2b1f7d4a5f7d929ee04e52298c35a7fe9c4e5a29580691b364i0",
|
|
66
|
+
# "num"=>"13070954",
|
|
67
|
+
# "date"=>"21.6.2023, 20:15:12",
|
|
68
|
+
# "address"=>"bc1qjhu...5lla3hj",
|
|
69
|
+
# "text"=>"{↵ \"p\": \"orc-721\",↵ \"op\": \"deploy\",..."},
|
|
70
|
+
# {"id"=>"157fdb5ebe82c12cefb6d48356d79bf5646e267980a2234294a8a330a4234be7i0",
|
|
71
|
+
# "num"=>"13074990",
|
|
72
|
+
# "date"=>"21.6.2023, 21:18:02",
|
|
73
|
+
# "address"=>"bc1qjhu...5lla3hj",
|
|
74
|
+
# "text"=>"{↵ \"p\":\"orc-721\",↵ \"op\":\"mint\",..."},
|
|
75
|
+
# ...]
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
Note: For now all results get "automagically" cached in `unisat/` indexd
|
|
80
|
+
by query (key) and pages. Example:
|
|
81
|
+
|
|
82
|
+
```
|
|
83
|
+
/unisat
|
|
84
|
+
/diyphunks
|
|
85
|
+
1.json
|
|
86
|
+
/orangepixels
|
|
87
|
+
1.json
|
|
88
|
+
2.json
|
|
89
|
+
3.json
|
|
90
|
+
4.json
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
That's it.
|
|
18
95
|
|
|
19
96
|
|
|
20
97
|
|
data/Rakefile
CHANGED
|
@@ -5,7 +5,7 @@ Hoe.spec 'unisat' do
|
|
|
5
5
|
|
|
6
6
|
self.version = Unisat::VERSION
|
|
7
7
|
|
|
8
|
-
self.summary = "unisat - lite helpers for full-text search in bitcoin ordinal inscriptions
|
|
8
|
+
self.summary = "unisat - unisat.io lite helpers for full-text search in bitcoin ordinal inscriptions"
|
|
9
9
|
self.description = summary
|
|
10
10
|
|
|
11
11
|
self.urls = { home: 'https://github.com/ordbase/ordbase' }
|
data/lib/unisat/cache.rb
CHANGED
|
@@ -40,6 +40,7 @@ def self.parse_page( html )
|
|
|
40
40
|
end # method _parse_page
|
|
41
41
|
|
|
42
42
|
|
|
43
|
+
|
|
43
44
|
class Cache
|
|
44
45
|
def initialize( dir )
|
|
45
46
|
@dir = dir
|
|
@@ -57,52 +58,68 @@ class Cache
|
|
|
57
58
|
results = read_json( path )
|
|
58
59
|
puts "==> #{i+1}/#{paths.size} - #{results.size} inscribe(s) in >#{path}<..."
|
|
59
60
|
|
|
60
|
-
results
|
|
61
|
-
date = h['date'] # note: might include unconfirmed!!!
|
|
62
|
-
## always "auto-magically" filter out unconfirmed for now
|
|
63
|
-
if date == 'unconfirmed'
|
|
64
|
-
puts " !! INFO - skipping unconfirmed inscribe"
|
|
65
|
-
next
|
|
66
|
-
end
|
|
67
|
-
|
|
68
|
-
## todo/fix:
|
|
69
|
-
## reformat date here/parse
|
|
70
|
-
## and change to iso e.g. 2023-12-24 17:18 or such - why? why not!!!!
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
## id -- split href by / and get last part (is id)
|
|
74
|
-
id = h['href'].split( '/')[-1]
|
|
75
|
-
## num -- remove/ strip leadingnumber sign (#)
|
|
76
|
-
num = h['num'].sub( '#', '' )
|
|
77
|
-
recs << {
|
|
78
|
-
'id' => id,
|
|
79
|
-
'num' => num,
|
|
80
|
-
'date' => date,
|
|
81
|
-
'address' => h['address'],
|
|
82
|
-
'text' => h['name'], ## change name to text
|
|
83
|
-
}
|
|
84
|
-
end
|
|
61
|
+
recs += _normalize( results )
|
|
85
62
|
end
|
|
86
63
|
puts " #{recs.size} inscribe(s) - total"
|
|
87
64
|
recs
|
|
88
65
|
end # method read
|
|
89
66
|
|
|
90
67
|
|
|
68
|
+
def _normalize( results )
|
|
69
|
+
recs = []
|
|
70
|
+
results.each do |h|
|
|
71
|
+
date = h['date'] # note: might include unconfirmed!!!
|
|
72
|
+
## always "auto-magically" filter out unconfirmed for now
|
|
73
|
+
if date == 'unconfirmed'
|
|
74
|
+
puts " !! INFO - skipping unconfirmed inscribe"
|
|
75
|
+
next
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
## todo/fix:
|
|
79
|
+
## reformat date here/parse
|
|
80
|
+
## and change to iso e.g. 2023-12-24 17:18 or such - why? why not!!!!
|
|
81
|
+
|
|
82
|
+
## id -- split href by / and get last part (is id)
|
|
83
|
+
id = h['href'].split( '/')[-1]
|
|
84
|
+
## num -- remove/ strip leadingnumber sign (#)
|
|
85
|
+
num = h['num'].sub( '#', '' )
|
|
86
|
+
recs << {
|
|
87
|
+
'id' => id,
|
|
88
|
+
'num' => num,
|
|
89
|
+
'date' => date,
|
|
90
|
+
'address' => h['address'],
|
|
91
|
+
'text' => h['name'], ## change name to text
|
|
92
|
+
}
|
|
93
|
+
end
|
|
94
|
+
recs
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
|
|
91
98
|
def exist?( key, offset: )
|
|
92
99
|
outpath = "#{@dir}/#{key}/#{offset}.json"
|
|
93
100
|
File.exist?( outpath )
|
|
94
101
|
end
|
|
95
102
|
|
|
103
|
+
def read_page( key, offset: )
|
|
104
|
+
outpath = "#{@dir}/#{key}/#{offset}.json"
|
|
105
|
+
results = read_json( outpath )
|
|
106
|
+
recs = _normalize( results )
|
|
107
|
+
recs
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
|
|
96
111
|
def add_page( page, key, offset: )
|
|
97
|
-
|
|
112
|
+
results = Unisat.parse_page( page )
|
|
98
113
|
|
|
99
114
|
## note: only write if results > 0
|
|
100
|
-
if
|
|
115
|
+
if results.size > 0
|
|
101
116
|
outpath = "#{@dir}/#{key}/#{offset}.json"
|
|
102
|
-
write_json( outpath,
|
|
117
|
+
write_json( outpath, results )
|
|
103
118
|
else
|
|
104
119
|
puts "!! WARN - no results found in page #{offset} for key >#{key}<"
|
|
105
120
|
end
|
|
121
|
+
recs = _normalize( results )
|
|
122
|
+
recs
|
|
106
123
|
end
|
|
107
124
|
end # class Cache
|
|
108
125
|
end # module Unisat
|
data/lib/unisat/version.rb
CHANGED
data/lib/unisat.rb
CHANGED
|
@@ -40,30 +40,48 @@ def self.cache() Unisat.cache; end
|
|
|
40
40
|
###
|
|
41
41
|
# search
|
|
42
42
|
def self.search( q, offset: 1,
|
|
43
|
-
|
|
44
|
-
|
|
43
|
+
force: false,
|
|
44
|
+
limit: nil )
|
|
45
45
|
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
46
|
+
browser = nil
|
|
47
|
+
page = nil
|
|
48
|
+
recs = []
|
|
49
49
|
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
50
|
+
print "==> searching >#{q}< starting at page #{offset}"
|
|
51
|
+
print " up to #{offset+limit-1} (#{limit} pages)" if limit && limit > 1
|
|
52
|
+
print ", force: true" if force
|
|
53
|
+
print "...\n"
|
|
54
54
|
|
|
55
|
-
|
|
55
|
+
page_recs = nil
|
|
56
|
+
i = 0
|
|
57
|
+
loop do
|
|
58
|
+
break if limit && i >= limit
|
|
56
59
|
count = offset+i
|
|
57
60
|
|
|
58
61
|
page_url = "https://unisat.io/search?q=#{q}&type=text&p=#{count}"
|
|
59
62
|
print page_url
|
|
60
63
|
|
|
61
64
|
## check if already in cache
|
|
62
|
-
if force == false && cache.exist?( q,
|
|
65
|
+
if force == false && cache.exist?( q,
|
|
66
|
+
offset: count )
|
|
63
67
|
print "...in cache\n"
|
|
68
|
+
page_recs = cache.read_page( q, offset: count )
|
|
69
|
+
recs += page_recs
|
|
70
|
+
|
|
71
|
+
break if limit.nil? && page_recs.size < 32 ## check for auto-limit (if less than 32 recs)
|
|
72
|
+
i += 1
|
|
64
73
|
next
|
|
65
74
|
end
|
|
66
75
|
|
|
76
|
+
if browser.nil? ## first page request? launch browser on demand
|
|
77
|
+
opts = {}
|
|
78
|
+
opts[:headless] = false
|
|
79
|
+
opts[:executable_path] = chrome_path if chrome_path ## add only if set (default is nil)
|
|
80
|
+
|
|
81
|
+
browser = ::Puppeteer.launch( **opts )
|
|
82
|
+
page = browser.new_page
|
|
83
|
+
end
|
|
84
|
+
|
|
67
85
|
print "... goto page ...\n"
|
|
68
86
|
response = page.goto( page_url )
|
|
69
87
|
pp response.headers
|
|
@@ -77,16 +95,7 @@ def self.search( q, offset: 1,
|
|
|
77
95
|
puts
|
|
78
96
|
puts el.evaluate("el => el.innerText")
|
|
79
97
|
#=> Searched for "biixel" among 3252336 records, found 1105 results.
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
## print pagination
|
|
83
|
-
page.wait_for_selector('ul.ant-pagination')
|
|
84
|
-
el = page.query_selector("ul.ant-pagination")
|
|
85
|
-
puts
|
|
86
|
-
puts el.evaluate("el => el.innerText")
|
|
87
|
-
#=> 1 ••• 32 33 34 35
|
|
88
|
-
|
|
89
|
-
|
|
98
|
+
|
|
90
99
|
## get search results (32 inscribes per page)
|
|
91
100
|
page.wait_for_selector('div.sats-list')
|
|
92
101
|
el = page.query_selector("div.sats-list")
|
|
@@ -94,14 +103,32 @@ def self.search( q, offset: 1,
|
|
|
94
103
|
html = el.evaluate("el => el.innerHTML")
|
|
95
104
|
# puts
|
|
96
105
|
# puts html
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
106
|
+
|
|
107
|
+
## print pagination
|
|
108
|
+
## note: only available if more than one page!!!
|
|
109
|
+
el = page.query_selector("ul.ant-pagination")
|
|
110
|
+
if el
|
|
111
|
+
puts
|
|
112
|
+
puts el.evaluate("el => el.innerText")
|
|
113
|
+
#=> 1 ••• 32 33 34 35
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
page_recs = cache.add_page( html, q,
|
|
117
|
+
offset: count )
|
|
118
|
+
recs += page_recs
|
|
119
|
+
|
|
120
|
+
break if limit.nil? && page_recs.size < 32 ## check for auto-limit (if less than 32 recs)
|
|
121
|
+
|
|
122
|
+
i += 1
|
|
123
|
+
end # loop
|
|
124
|
+
|
|
125
|
+
if browser
|
|
126
|
+
puts "sleeping 2 secs before browser shutdown..."
|
|
127
|
+
sleep( 2 )
|
|
128
|
+
browser.close
|
|
100
129
|
end
|
|
101
130
|
|
|
102
|
-
|
|
103
|
-
sleep( 2 )
|
|
104
|
-
end
|
|
131
|
+
recs
|
|
105
132
|
end # method search
|
|
106
133
|
end # module Puppeteer
|
|
107
134
|
end # module Unisat
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: unisat
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.0
|
|
4
|
+
version: 0.1.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Gerald Bauer
|
|
@@ -86,8 +86,8 @@ dependencies:
|
|
|
86
86
|
- - "~>"
|
|
87
87
|
- !ruby/object:Gem::Version
|
|
88
88
|
version: '4.0'
|
|
89
|
-
description: unisat - lite helpers for full-text search in bitcoin ordinal
|
|
90
|
-
|
|
89
|
+
description: unisat - unisat.io lite helpers for full-text search in bitcoin ordinal
|
|
90
|
+
inscriptions
|
|
91
91
|
email: gerald.bauer@gmail.com
|
|
92
92
|
executables: []
|
|
93
93
|
extensions: []
|
|
@@ -127,6 +127,5 @@ requirements: []
|
|
|
127
127
|
rubygems_version: 3.4.10
|
|
128
128
|
signing_key:
|
|
129
129
|
specification_version: 4
|
|
130
|
-
summary: unisat - lite helpers for full-text search in bitcoin ordinal inscriptions
|
|
131
|
-
powered by unisat.io
|
|
130
|
+
summary: unisat - unisat.io lite helpers for full-text search in bitcoin ordinal inscriptions
|
|
132
131
|
test_files: []
|