fuse_link_fs 0.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/Gemfile +6 -0
- data/Gemfile.lock +62 -0
- data/README.md +0 -0
- data/bin/fuselinkfs +187 -0
- data/fuse_link_fs.gemspec +31 -0
- data/lib/fuse-link-fs.rb +1 -0
- data/lib/fuse_link_fs/configuration.rb +41 -0
- data/lib/fuse_link_fs/provider.rb +137 -0
- data/lib/fuse_link_fs/storage.rb +164 -0
- data/lib/fuse_link_fs/version.rb +3 -0
- data/lib/fuse_link_fs.rb +20 -0
- metadata +166 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: ee22052a8b360bcfa81db486ff3df4d444eebf385e11104f2fc65eee65d47b59
|
4
|
+
data.tar.gz: 4ca69b8d9fb2bf4f58abed68d411ce22afda8f825e1d2d9adb8024aeec26ee1e
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 31b099731494823480b3f094a2c5d97e1dd8af345ed6d8757f2759ea385ee1b1edc646cb2a89451564d8b63c8c88e6a828b360950f00496906662e278d764060
|
7
|
+
data.tar.gz: bd044dbb4c0a1294cd1120770170a1f31cb92a34838f5c12cf4a945eb09a5b6041872cfbf2b75f57528202cfb627292f9da90aac87ee614720003063149eaae3
|
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
@@ -0,0 +1,62 @@
|
|
1
|
+
PATH
|
2
|
+
remote: .
|
3
|
+
specs:
|
4
|
+
fuse_link_fs (0.0.3)
|
5
|
+
activesupport (~> 6.0)
|
6
|
+
ffi-libfuse
|
7
|
+
logger
|
8
|
+
socksify
|
9
|
+
zeitwerk
|
10
|
+
|
11
|
+
GEM
|
12
|
+
remote: https://rubygems.org/
|
13
|
+
specs:
|
14
|
+
activesupport (6.1.7.10)
|
15
|
+
concurrent-ruby (~> 1.0, >= 1.0.2)
|
16
|
+
i18n (>= 1.6, < 2)
|
17
|
+
minitest (>= 5.1)
|
18
|
+
tzinfo (~> 2.0)
|
19
|
+
zeitwerk (~> 2.3)
|
20
|
+
byebug (12.0.0)
|
21
|
+
concurrent-ruby (1.3.5)
|
22
|
+
diff-lcs (1.6.2)
|
23
|
+
ffi (1.17.2)
|
24
|
+
ffi (1.17.2-x86_64-linux-gnu)
|
25
|
+
ffi-libfuse (0.4.4)
|
26
|
+
ffi (~> 1)
|
27
|
+
i18n (1.14.7)
|
28
|
+
concurrent-ruby (~> 1.0)
|
29
|
+
logger (1.7.0)
|
30
|
+
minitest (5.25.5)
|
31
|
+
rspec (3.13.1)
|
32
|
+
rspec-core (~> 3.13.0)
|
33
|
+
rspec-expectations (~> 3.13.0)
|
34
|
+
rspec-mocks (~> 3.13.0)
|
35
|
+
rspec-core (3.13.5)
|
36
|
+
rspec-support (~> 3.13.0)
|
37
|
+
rspec-expectations (3.13.5)
|
38
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
39
|
+
rspec-support (~> 3.13.0)
|
40
|
+
rspec-mocks (3.13.5)
|
41
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
42
|
+
rspec-support (~> 3.13.0)
|
43
|
+
rspec-support (3.13.5)
|
44
|
+
rspec_junit_formatter (0.6.0)
|
45
|
+
rspec-core (>= 2, < 4, != 2.12.0)
|
46
|
+
socksify (1.8.1)
|
47
|
+
tzinfo (2.0.6)
|
48
|
+
concurrent-ruby (~> 1.0)
|
49
|
+
zeitwerk (2.7.3)
|
50
|
+
|
51
|
+
PLATFORMS
|
52
|
+
ruby
|
53
|
+
x86_64-linux
|
54
|
+
|
55
|
+
DEPENDENCIES
|
56
|
+
byebug
|
57
|
+
fuse_link_fs!
|
58
|
+
rspec (~> 3.0)
|
59
|
+
rspec_junit_formatter
|
60
|
+
|
61
|
+
BUNDLED WITH
|
62
|
+
2.6.2
|
data/README.md
ADDED
File without changes
|
data/bin/fuselinkfs
ADDED
@@ -0,0 +1,187 @@
|
|
1
|
+
#!/bin/env ruby
|
2
|
+
|
3
|
+
require 'logger'
|
4
|
+
require 'active_support/all'
|
5
|
+
require 'ffi/libfuse'
|
6
|
+
require 'fuse_link_fs'
|
7
|
+
require 'socksify'
|
8
|
+
|
9
|
+
UTIL = File.basename(__FILE__)
|
10
|
+
SELF = File.absolute_path(__FILE__)
|
11
|
+
|
12
|
+
Thread.abort_on_exception = true
|
13
|
+
|
14
|
+
[STDIN, STDOUT, STDERR].each do |io|
|
15
|
+
io.sync = true
|
16
|
+
rescue StandardError
|
17
|
+
end
|
18
|
+
|
19
|
+
class FuseLinkFs::Fs
|
20
|
+
include FFI::Libfuse::Adapter::Ruby
|
21
|
+
include FFI::Libfuse::Adapter::Fuse2Compat
|
22
|
+
|
23
|
+
# FUSE Configuration methods
|
24
|
+
|
25
|
+
def logger
|
26
|
+
FuseLinkFs.config.logger
|
27
|
+
end
|
28
|
+
|
29
|
+
def fuse_configure *_args, **_kwargs
|
30
|
+
if (addr = ENV.fetch('http_proxy', nil))
|
31
|
+
uri = URI(addr)
|
32
|
+
puts "USING PROXY: #{uri}"
|
33
|
+
TCPSocket.socks_server = uri.hostname
|
34
|
+
TCPSocket.socks_port = uri.port
|
35
|
+
end
|
36
|
+
|
37
|
+
ARGV.unshift '-unused' # надо чтоб распарсить mountpoint
|
38
|
+
@mountpoint = FFI::Libfuse::Main.fuse_parse_cmdline(args: ARGV, handler: nil)[:mountpoint]
|
39
|
+
|
40
|
+
@db_path = File.join(@mountpoint, '.fuselinks')
|
41
|
+
@db_file = File.open(@db_path, File::RDWR | File::CREAT)
|
42
|
+
@db = begin
|
43
|
+
JSON.parse(@db_file.read)
|
44
|
+
rescue StandardError
|
45
|
+
{}
|
46
|
+
end
|
47
|
+
@storage = FuseLinkFs::Storage.new(FuseLinkFs::Provider::LIST.map(&:new), cache: {})
|
48
|
+
|
49
|
+
FuseLinkFs.configure do |config|
|
50
|
+
config.logger = ActiveSupport::TaggedLogging.new(::Logger.new(STDERR, formatter: Logger::Formatter.new))
|
51
|
+
config.logger.level = ENV.fetch('LOG_LEVEL', :info)
|
52
|
+
config.logger.level = :debug if debug?
|
53
|
+
end
|
54
|
+
rescue StandardError => e
|
55
|
+
logger.error "fuse_configure exception: #{e}"
|
56
|
+
raise
|
57
|
+
end
|
58
|
+
|
59
|
+
# FUSE callbacks
|
60
|
+
|
61
|
+
def getattr(path, stat, *args)
|
62
|
+
logger.debug "[getattr] path=#{path} stat=#{stat.inspect} args=#{args}"
|
63
|
+
|
64
|
+
case path
|
65
|
+
when '/'
|
66
|
+
stat.directory(mode: 0o777)
|
67
|
+
nil
|
68
|
+
else
|
69
|
+
file = get_file(path)
|
70
|
+
raise Errno::ENOENT if file.nil?
|
71
|
+
|
72
|
+
return stat.file(mode: 0o744, size: 0) if file.empty?
|
73
|
+
|
74
|
+
stat.file(mode: file.fetch('perms', 0o744), size: file.fetch('size', file['content'].to_s.size))
|
75
|
+
end
|
76
|
+
rescue StandardError => e
|
77
|
+
logger.error "[getattr] exception: #{e}"
|
78
|
+
raise
|
79
|
+
end
|
80
|
+
|
81
|
+
def readdir(path, *args)
|
82
|
+
logger.debug "[readdir] path=#{path} args=#{args}"
|
83
|
+
|
84
|
+
@db.fetch('files', {}).each do |(p, _json)|
|
85
|
+
yield p[1..-1] # убираем лидирующий /
|
86
|
+
end
|
87
|
+
rescue StandardError => e
|
88
|
+
logger.error "[readdir] exception: #{e}"
|
89
|
+
raise
|
90
|
+
end
|
91
|
+
|
92
|
+
def open(path, *args)
|
93
|
+
logger.debug "[open] path=#{path} args=#{args}"
|
94
|
+
rescue StandardError => e
|
95
|
+
logger.error "[open] exception: #{e}"
|
96
|
+
raise
|
97
|
+
end
|
98
|
+
|
99
|
+
def release(path, *args, **kwargs)
|
100
|
+
logger.debug "[release] args=#{args} kwargs=#{kwargs}"
|
101
|
+
file = get_file(path)
|
102
|
+
|
103
|
+
if (content = file.fetch('content', nil))
|
104
|
+
link = @storage.store(content)
|
105
|
+
file['link'] = link
|
106
|
+
file['size'] = content.size
|
107
|
+
file.delete('content')
|
108
|
+
end
|
109
|
+
rescue StandardError => e
|
110
|
+
logger.error "[release] exception: #{e}"
|
111
|
+
raise
|
112
|
+
end
|
113
|
+
|
114
|
+
def write_buf(path, offset, _buffer, *args, &block)
|
115
|
+
logger.debug "[write_buf] path=#{path} offset=#{offset} args=#{args}"
|
116
|
+
|
117
|
+
data = block.call
|
118
|
+
file = get_file(path)
|
119
|
+
file['content'] ||= ''
|
120
|
+
file['content'] += data
|
121
|
+
|
122
|
+
data.size
|
123
|
+
rescue StandardError => e
|
124
|
+
logger.error "[write_buf] exception: #{e}"
|
125
|
+
raise
|
126
|
+
end
|
127
|
+
|
128
|
+
# Creates the File at {#map_path}(path)
|
129
|
+
def create(path, perms = 0o644, ffi = nil)
|
130
|
+
logger.debug "[create] path=#{path} perms=#{perms} ffi=#{ffi.inspect}"
|
131
|
+
put_file(path, { perms: perms })
|
132
|
+
rescue StandardError => e
|
133
|
+
puts "[create] exception: #{e}"
|
134
|
+
raise
|
135
|
+
end
|
136
|
+
|
137
|
+
def unlink(path)
|
138
|
+
logger.debug "[unlink] path=#{path}"
|
139
|
+
get_file!(path)
|
140
|
+
|
141
|
+
remove_file!(path)
|
142
|
+
end
|
143
|
+
|
144
|
+
def read(path, *args)
|
145
|
+
logger.debug "[read] path=#{path} args=#{args}"
|
146
|
+
file = get_file!(path)
|
147
|
+
|
148
|
+
if (link = file.fetch('link', link))
|
149
|
+
FuseLinkFs::Provider.dec(@storage.extract(link))
|
150
|
+
else
|
151
|
+
file.fetch('content')
|
152
|
+
end
|
153
|
+
rescue StandardError => e
|
154
|
+
logger.debug "[read] exception: #{e}"
|
155
|
+
raise
|
156
|
+
end
|
157
|
+
|
158
|
+
def destroy *args, **kwargs
|
159
|
+
logger.debug "[destroy] args=#{args} kwargs=#{kwargs}"
|
160
|
+
@db_file.rewind
|
161
|
+
@db_file.write(@db.to_json)
|
162
|
+
@db_file.close
|
163
|
+
end
|
164
|
+
|
165
|
+
def get_file(path)
|
166
|
+
@db.fetch('files', {}).fetch(path, nil)
|
167
|
+
end
|
168
|
+
|
169
|
+
def get_file!(path)
|
170
|
+
get_file(path).tap do |file|
|
171
|
+
raise Errno::ENOENT if file.nil?
|
172
|
+
end
|
173
|
+
end
|
174
|
+
|
175
|
+
def put_file(path, json)
|
176
|
+
@db['files'] ||= {}
|
177
|
+
@db['files'][path] = json
|
178
|
+
end
|
179
|
+
|
180
|
+
def remove_file!(path)
|
181
|
+
@db['files'].delete(path)
|
182
|
+
end
|
183
|
+
end
|
184
|
+
|
185
|
+
exit(FFI::Libfuse.fuse_main(operations: FuseLinkFs::Fs.new)) if __FILE__ == $0
|
186
|
+
|
187
|
+
exit(0)
|
@@ -0,0 +1,31 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
lib = File.expand_path('lib', __dir__)
|
4
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
5
|
+
require 'fuse_link_fs/version'
|
6
|
+
|
7
|
+
Gem::Specification.new do |spec|
|
8
|
+
spec.name = 'fuse_link_fs'
|
9
|
+
spec.version = ENV['BUILDVERSION'].to_i > 0 ? "#{FuseLinkFs::VERSION}.#{ENV['BUILDVERSION'].to_i}" : FuseLinkFs::VERSION
|
10
|
+
spec.authors = ['Samoilenko Yuri']
|
11
|
+
spec.email = ['kinnalru@gmail.com']
|
12
|
+
|
13
|
+
spec.summary = 'Fuse Filesystem backed by link shortify'
|
14
|
+
spec.description = 'Fuse Filesystem backed by link shortify'
|
15
|
+
|
16
|
+
spec.files = Dir['bin/*', 'lib/**/*', 'Gemfile*', 'LICENSE.txt', 'README.md', '*.gemspec']
|
17
|
+
spec.bindir = 'bin'
|
18
|
+
spec.require_paths = ['lib']
|
19
|
+
spec.executables = ['fuselinkfs']
|
20
|
+
|
21
|
+
spec.add_runtime_dependency 'activesupport', '~> 6.0'
|
22
|
+
spec.add_runtime_dependency 'logger'
|
23
|
+
spec.add_runtime_dependency 'ffi-libfuse'
|
24
|
+
spec.add_runtime_dependency 'zeitwerk'
|
25
|
+
spec.add_runtime_dependency 'socksify'
|
26
|
+
|
27
|
+
spec.add_development_dependency 'rspec', '~> 3.0'
|
28
|
+
spec.add_development_dependency 'rspec_junit_formatter'
|
29
|
+
|
30
|
+
spec.add_development_dependency 'byebug'
|
31
|
+
end
|
data/lib/fuse-link-fs.rb
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
require_relative './fuse_link_fs'
|
@@ -0,0 +1,41 @@
|
|
1
|
+
require 'uri'
|
2
|
+
|
3
|
+
module FuseLinkFs
|
4
|
+
class Configuration
|
5
|
+
attr_accessor :logger
|
6
|
+
|
7
|
+
def initialize(logger = ::Logger.new(STDOUT, formatter: Logger::Formatter.new))
|
8
|
+
@logger = logger.respond_to?(:tagged) ? logger : ActiveSupport::TaggedLogging.new(logger)
|
9
|
+
end
|
10
|
+
|
11
|
+
module Concern
|
12
|
+
extend ActiveSupport::Concern
|
13
|
+
|
14
|
+
included do |_klass|
|
15
|
+
end
|
16
|
+
|
17
|
+
class_methods do
|
18
|
+
# Instantiate the Configuration singleton
|
19
|
+
# or return it. Remember that the instance
|
20
|
+
# has attribute readers so that we can access
|
21
|
+
# the configured values
|
22
|
+
def configuration
|
23
|
+
@configuration ||= FuseLinkFs::Configuration.new
|
24
|
+
end
|
25
|
+
|
26
|
+
def config
|
27
|
+
configuration
|
28
|
+
end
|
29
|
+
|
30
|
+
# This is the configure block definition.
|
31
|
+
# The configuration method will return the
|
32
|
+
# Configuration singleton, which is then yielded
|
33
|
+
# to the configure block. Then it's just a matter
|
34
|
+
# of using the attribute accessors we previously defined
|
35
|
+
def configure
|
36
|
+
yield(configuration)
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
@@ -0,0 +1,137 @@
|
|
1
|
+
require 'base64'
|
2
|
+
require 'cgi'
|
3
|
+
require 'net/http'
|
4
|
+
require 'socksify/http'
|
5
|
+
|
6
|
+
module FuseLinkFs
|
7
|
+
class Provider
|
8
|
+
def self.mk_data_uri(encoded_chunk, type:, size:, idx:)
|
9
|
+
"http://data.local?#{type}=#{encoded_chunk}&sz=#{size}&i=#{idx}"
|
10
|
+
end
|
11
|
+
|
12
|
+
def self.enc(data)
|
13
|
+
#Base64.strict_encode64(Base64.strict_encode64(data).strip).strip
|
14
|
+
Base64.strict_encode64(data).strip
|
15
|
+
end
|
16
|
+
|
17
|
+
def self.dec(data)
|
18
|
+
#Base64.decode64(Base64.decode64(data).strip).strip
|
19
|
+
Base64.decode64(data).strip
|
20
|
+
end
|
21
|
+
|
22
|
+
def match?(*args, **kwargs)
|
23
|
+
self.class.match?(*args, **kwargs)
|
24
|
+
end
|
25
|
+
|
26
|
+
def base
|
27
|
+
URI(self.class.base.to_s)
|
28
|
+
end
|
29
|
+
|
30
|
+
def store_raw(raw)
|
31
|
+
store(self.class.mk_data_uri(Base64.strict_encode64(raw).strip, type: 'chunk', size: raw.size))
|
32
|
+
end
|
33
|
+
|
34
|
+
def make_request(request)
|
35
|
+
with_connection(request.uri) do |http|
|
36
|
+
http.request request
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
def extract(link)
|
41
|
+
make_request(Net::HTTP::Get.new(URI(link.to_s))).fetch('Location')
|
42
|
+
end
|
43
|
+
|
44
|
+
def with_connection(uri, &block)
|
45
|
+
Net::HTTP.start(uri.host, uri.port, read_timeout: 1, open_timeout: 1, use_ssl: uri.scheme == 'https', verify_mode: OpenSSL::SSL::VERIFY_NONE, &block)
|
46
|
+
end
|
47
|
+
|
48
|
+
LIST = []
|
49
|
+
end
|
50
|
+
|
51
|
+
class Provider::GooSu < Provider
|
52
|
+
def self.base
|
53
|
+
@base ||= URI('https://goo.su/frontend-api/convert').freeze
|
54
|
+
end
|
55
|
+
|
56
|
+
def self.match?(link)
|
57
|
+
!!link.to_s['https://goo.su']
|
58
|
+
end
|
59
|
+
|
60
|
+
def store(data_uri)
|
61
|
+
puts "stoging data_uri:#{data_uri}"
|
62
|
+
req = Net::HTTP::Post.new(base)
|
63
|
+
req['User-Agent'] =
|
64
|
+
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36'
|
65
|
+
req['Accept'] = 'application/json'
|
66
|
+
req.body = { url: data_uri.to_s, is_public: 1 }.to_json
|
67
|
+
req.content_type = 'application/json'
|
68
|
+
|
69
|
+
response = make_request(req)
|
70
|
+
response.value
|
71
|
+
puts response.body
|
72
|
+
JSON(response.body).fetch('short_url')
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
class Provider::ClckRu < Provider
|
77
|
+
def self.base
|
78
|
+
@base ||= URI('https://clck.ru/--').freeze
|
79
|
+
end
|
80
|
+
|
81
|
+
def self.match?(link)
|
82
|
+
!!link.to_s['https://clck.ru']
|
83
|
+
end
|
84
|
+
|
85
|
+
def store(data_uri)
|
86
|
+
uri = base
|
87
|
+
uri.query = { url: data_uri }.to_query
|
88
|
+
make_request(Net::HTTP::Get.new(uri)).body
|
89
|
+
end
|
90
|
+
|
91
|
+
def extract(link)
|
92
|
+
location = super
|
93
|
+
CGI.parse(URI(location).query).fetch('url').first
|
94
|
+
rescue StandardError => e
|
95
|
+
puts "Unable to extract from #{link}(#{location}):#{e}"
|
96
|
+
raise
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
class Provider::Kontentino < Provider
|
101
|
+
def self.base
|
102
|
+
@base ||= URI('https://kntn.ly/graphql').freeze
|
103
|
+
end
|
104
|
+
|
105
|
+
def self.match?(link)
|
106
|
+
!!link.to_s['https://kntn.ly']
|
107
|
+
end
|
108
|
+
|
109
|
+
def store(data_uri)
|
110
|
+
puts "stoging data_uri:#{data_uri}"
|
111
|
+
req = Net::HTTP::Post.new(base)
|
112
|
+
req['User-Agent'] =
|
113
|
+
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36'
|
114
|
+
req['Accept'] = 'application/json'
|
115
|
+
req.body = {
|
116
|
+
"query": "\n mutation createLinkMutation($input: NewLink!) {\n createLink(input: $input) {\n id\n shortUrl\n }\n }\n ",
|
117
|
+
"variables": {
|
118
|
+
"input": {
|
119
|
+
"url": data_uri,
|
120
|
+
"id": '',
|
121
|
+
"domain": 'https://kntn.ly',
|
122
|
+
"name": ''
|
123
|
+
}
|
124
|
+
},
|
125
|
+
"operationName": 'createLinkMutation'
|
126
|
+
}.to_json
|
127
|
+
req.content_type = 'application/json'
|
128
|
+
|
129
|
+
response = make_request(req)
|
130
|
+
response.value
|
131
|
+
(JSON(response.body).dig('data', 'createLink') || {}).fetch('shortUrl')
|
132
|
+
end
|
133
|
+
end
|
134
|
+
|
135
|
+
#Provider::LIST << Provider::ClckRu << Provider::GooSu << Provider::Kontentino
|
136
|
+
Provider::LIST << Provider::GooSu << Provider::Kontentino
|
137
|
+
end
|
@@ -0,0 +1,164 @@
|
|
1
|
+
require 'base64'
|
2
|
+
require 'cgi'
|
3
|
+
require 'net/http'
|
4
|
+
|
5
|
+
module FuseLinkFs
|
6
|
+
class RetryCaptchaError < StandardError; end
|
7
|
+
|
8
|
+
class Storage
|
9
|
+
def initialize(providers, cache: nil)
|
10
|
+
@providers = providers
|
11
|
+
@cache = cache
|
12
|
+
end
|
13
|
+
|
14
|
+
def random_provider
|
15
|
+
@providers.sample
|
16
|
+
end
|
17
|
+
|
18
|
+
def provider_for(link)
|
19
|
+
@providers.find { |p| p.class.match?(link) }
|
20
|
+
end
|
21
|
+
|
22
|
+
def store(data)
|
23
|
+
size = data.size
|
24
|
+
encoded = Provider.enc(data)
|
25
|
+
|
26
|
+
links = encoded.scan(/.{1,1900}/).each_with_index.map do |encoded_chunk, idx|
|
27
|
+
store_to_link(encoded_chunk, type: 'chunk', size: size, idx: idx)
|
28
|
+
end
|
29
|
+
|
30
|
+
store_links_if_needed(links, size: size).tap do |result|
|
31
|
+
puts "RESULT:#{result}"
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def analyze(link)
|
36
|
+
list = { link => {} }
|
37
|
+
|
38
|
+
puts "ANALYZE #{link}"
|
39
|
+
|
40
|
+
data_uri = with_retry do
|
41
|
+
provider_for(link).extract(link)
|
42
|
+
end
|
43
|
+
|
44
|
+
puts " data_uri: #{data_uri}"
|
45
|
+
|
46
|
+
|
47
|
+
#params = CGI.parse(URI(data_uri).query)
|
48
|
+
params = URI(data_uri).query.split('&').each_with_object({}) do |s, o|
|
49
|
+
k,v=s.split('=')
|
50
|
+
o[k] = v
|
51
|
+
end
|
52
|
+
|
53
|
+
puts " params: #{params}"
|
54
|
+
type = if params.fetch('chunk', nil)
|
55
|
+
'chunk'
|
56
|
+
elsif params.fetch('links', nil)
|
57
|
+
'links'
|
58
|
+
else
|
59
|
+
raise raise "invalid TYPE[1]:#{type}"
|
60
|
+
end
|
61
|
+
|
62
|
+
if type == 'chunk'
|
63
|
+
list[link][:type] = 'chunk'
|
64
|
+
list[link][:params] = params
|
65
|
+
elsif type == 'links'
|
66
|
+
list[link][:type] = 'links'
|
67
|
+
list[link][:params] = params
|
68
|
+
links = Provider.dec(params.fetch(type)).split('|')
|
69
|
+
list[link][:links] = links.map { |lnk| analyze(lnk) }
|
70
|
+
else
|
71
|
+
raise "invalid TYPE:#{type}"
|
72
|
+
end
|
73
|
+
|
74
|
+
list
|
75
|
+
end
|
76
|
+
|
77
|
+
def extract(link)
|
78
|
+
puts "EXTRACT:#{link}"
|
79
|
+
return @cache[link] if @cache && @cache[link]
|
80
|
+
|
81
|
+
provider = provider_for(link)
|
82
|
+
|
83
|
+
data_uri = with_retry do
|
84
|
+
provider.extract(link)
|
85
|
+
end
|
86
|
+
|
87
|
+
#params = CGI.parse(URI(data_uri).query)
|
88
|
+
params = URI(data_uri).query.split('&').each_with_object({}) do |s, o|
|
89
|
+
k,v=s.split('=')
|
90
|
+
o[k] = v
|
91
|
+
end
|
92
|
+
|
93
|
+
type = if params.fetch('chunk', nil)
|
94
|
+
'chunk'
|
95
|
+
elsif params.fetch('links', nil)
|
96
|
+
'links'
|
97
|
+
else
|
98
|
+
raise raise "invalid TYPE[1]:#{type}"
|
99
|
+
end
|
100
|
+
data = params.fetch(type)
|
101
|
+
|
102
|
+
if type == 'chunk'
|
103
|
+
data.tap do |d|
|
104
|
+
@cache[link] = d if @cache
|
105
|
+
end
|
106
|
+
elsif type == 'links'
|
107
|
+
links = Provider.dec(data).split('|')
|
108
|
+
links.map do |lnk|
|
109
|
+
extract(lnk)
|
110
|
+
end.join.tap do |d|
|
111
|
+
@cache[link] = d if @cache
|
112
|
+
end
|
113
|
+
else
|
114
|
+
raise "invalid TYPE:#{type}"
|
115
|
+
end
|
116
|
+
end
|
117
|
+
|
118
|
+
def store_links_if_needed(links, size:)
|
119
|
+
return links.first if links.size <= 1
|
120
|
+
|
121
|
+
next_links = links.each_slice(30).each_with_index.map do |group, idx|
|
122
|
+
encoded = Provider.enc(group.join('|'))
|
123
|
+
store_to_link(encoded, type: 'links', size: size, idx: idx)
|
124
|
+
end
|
125
|
+
|
126
|
+
store_links_if_needed(next_links, size: size)
|
127
|
+
end
|
128
|
+
|
129
|
+
def store_to_link(encoded_chunk, type:, size:, idx:)
|
130
|
+
uri = Provider.mk_data_uri(encoded_chunk, type: type, size: size, idx: idx)
|
131
|
+
puts " store_to_link: #{uri} #{idx}"
|
132
|
+
with_retry do
|
133
|
+
random_provider.store(uri)
|
134
|
+
end
|
135
|
+
end
|
136
|
+
|
137
|
+
def with_retry(count = 5, delay: 0.1, klass: nil)
|
138
|
+
retries ||= 0
|
139
|
+
yield(retries)
|
140
|
+
rescue RetryCaptchaError => e
|
141
|
+
sleep(delay * 10 + (retries**2) * delay)
|
142
|
+
if (retries += 1) < count
|
143
|
+
logger.warn "Retry after error: #{e.inspect}. #{e.backtrace}" if respond_to?(:logger)
|
144
|
+
retry
|
145
|
+
else
|
146
|
+
raise if klass.nil?
|
147
|
+
return nil if klass == :skip
|
148
|
+
|
149
|
+
raise klass.new(e.message)
|
150
|
+
end
|
151
|
+
rescue StandardError => e
|
152
|
+
sleep(delay + (retries**2) * delay)
|
153
|
+
if (retries += 1) < count
|
154
|
+
logger.warn "Retry after error: #{e.inspect}. #{e.backtrace}" if respond_to?(:logger)
|
155
|
+
retry
|
156
|
+
else
|
157
|
+
raise if klass.nil?
|
158
|
+
return nil if klass == :skip
|
159
|
+
|
160
|
+
raise klass.new(e.message)
|
161
|
+
end
|
162
|
+
end
|
163
|
+
end
|
164
|
+
end
|
data/lib/fuse_link_fs.rb
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
require 'logger'
|
2
|
+
require 'active_support/all'
|
3
|
+
|
4
|
+
require 'zeitwerk'
|
5
|
+
|
6
|
+
loader = Zeitwerk::Loader.for_gem
|
7
|
+
loader.ignore("#{__dir__}/fuse-link-fs.rb")
|
8
|
+
loader.inflector.inflect(
|
9
|
+
'fuse_link_fs' => 'FuseLinkFs',
|
10
|
+
'fuse-link-fs' => 'FuseLinkFs'
|
11
|
+
)
|
12
|
+
loader.setup
|
13
|
+
|
14
|
+
module FuseLinkFs
|
15
|
+
include Configuration::Concern
|
16
|
+
|
17
|
+
def self.setup_logger(logger)
|
18
|
+
logger.respond_to?(:tagged) ? logger : ActiveSupport::TaggedLogging.new(logger)
|
19
|
+
end
|
20
|
+
end
|
metadata
ADDED
@@ -0,0 +1,166 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: fuse_link_fs
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.3
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Samoilenko Yuri
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2025-09-04 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: activesupport
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '6.0'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '6.0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: logger
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: ffi-libfuse
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: zeitwerk
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: socksify
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - ">="
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0'
|
76
|
+
type: :runtime
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - ">="
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: rspec
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - "~>"
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '3.0'
|
90
|
+
type: :development
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - "~>"
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '3.0'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: rspec_junit_formatter
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - ">="
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0'
|
104
|
+
type: :development
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - ">="
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '0'
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: byebug
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - ">="
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0'
|
118
|
+
type: :development
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - ">="
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: '0'
|
125
|
+
description: Fuse Filesystem backed by link shortify
|
126
|
+
email:
|
127
|
+
- kinnalru@gmail.com
|
128
|
+
executables:
|
129
|
+
- fuselinkfs
|
130
|
+
extensions: []
|
131
|
+
extra_rdoc_files: []
|
132
|
+
files:
|
133
|
+
- Gemfile
|
134
|
+
- Gemfile.lock
|
135
|
+
- README.md
|
136
|
+
- bin/fuselinkfs
|
137
|
+
- fuse_link_fs.gemspec
|
138
|
+
- lib/fuse-link-fs.rb
|
139
|
+
- lib/fuse_link_fs.rb
|
140
|
+
- lib/fuse_link_fs/configuration.rb
|
141
|
+
- lib/fuse_link_fs/provider.rb
|
142
|
+
- lib/fuse_link_fs/storage.rb
|
143
|
+
- lib/fuse_link_fs/version.rb
|
144
|
+
homepage:
|
145
|
+
licenses: []
|
146
|
+
metadata: {}
|
147
|
+
post_install_message:
|
148
|
+
rdoc_options: []
|
149
|
+
require_paths:
|
150
|
+
- lib
|
151
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
152
|
+
requirements:
|
153
|
+
- - ">="
|
154
|
+
- !ruby/object:Gem::Version
|
155
|
+
version: '0'
|
156
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
157
|
+
requirements:
|
158
|
+
- - ">="
|
159
|
+
- !ruby/object:Gem::Version
|
160
|
+
version: '0'
|
161
|
+
requirements: []
|
162
|
+
rubygems_version: 3.4.10
|
163
|
+
signing_key:
|
164
|
+
specification_version: 4
|
165
|
+
summary: Fuse Filesystem backed by link shortify
|
166
|
+
test_files: []
|