intosql 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +8 -0
- data/LICENSE +21 -0
- data/README.md +51 -0
- data/lib/intosql/client.rb +179 -0
- data/lib/intosql/conversions.rb +144 -0
- data/lib/intosql/errors.rb +58 -0
- data/lib/intosql/models.rb +80 -0
- data/lib/intosql/source.rb +32 -0
- data/lib/intosql/spec.rb +19 -0
- data/lib/intosql/transport.rb +75 -0
- data/lib/intosql/version.rb +5 -0
- data/lib/intosql/webhooks.rb +44 -0
- data/lib/intosql.rb +15 -0
- metadata +58 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: b676b287df23b8fe20bbcb97825e67771ac377d15b0bb92aeb74852c4e9e5dbe
|
|
4
|
+
data.tar.gz: 7486228ea1d9f8b5908c7b5d8a398f5a2c60d621d5984b13e6ac3af964fc552e
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: 3397f1e8a866a81eb52e64c6db766df8709f49b0fc3cab1b6cafef0b0f076df9e9e0b8c9f0b5a5a689c4aa18df3bc82d64383db7bdeff12363b928dcb7898494
|
|
7
|
+
data.tar.gz: 932b8af6b15e7ffab77f9f65ade11449eee62cfc553d36e7b8b1812115b61d1ad023210479e693d331ae803b950c192868c7341f23b4c891e19e032ac8cf9761
|
data/CHANGELOG.md
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## 0.1.0
|
|
4
|
+
|
|
5
|
+
- `convert` and `convert_url`, every conversion endpoint (`create`, `create_from_url`, `upload`, `start`, `retrieve`, `list`, `cancel`, `create_case`, `delete`, `wait`), `usage` and `spec`.
|
|
6
|
+
- `verify_webhook` and `parse_webhook`, which returns a typed `WebhookEvent`.
|
|
7
|
+
- `APIError` with `status`, `code`, `fields` and `request_id`; `ConversionFailedError`, `ConversionCanceledError` and `TimeoutError`.
|
|
8
|
+
- Retries for rate limits, a full account, server errors and lost connections, with the same idempotency key; never `already_started`.
|
data/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 IntoSQL
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
data/README.md
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
# IntoSQL for Ruby
|
|
2
|
+
|
|
3
|
+
Convert spreadsheets (XLSX, XLS, ODS), CSV and JSON into SQL for PostgreSQL, MySQL, MariaDB, SQLite, SQL Server or Oracle with the [IntoSQL API](https://intosql.com/developers). Column types are inferred and dirty values fixed, so the script loads on the first try.
|
|
4
|
+
|
|
5
|
+
```sh
|
|
6
|
+
gem install intosql
|
|
7
|
+
```
|
|
8
|
+
|
|
9
|
+
Ruby 3.1 or newer, no dependencies beyond the standard library. The API is part of IntoSQL Pro; make a key in Settings, API keys.
|
|
10
|
+
|
|
11
|
+
## Convert a file
|
|
12
|
+
|
|
13
|
+
```ruby
|
|
14
|
+
require "intosql"
|
|
15
|
+
|
|
16
|
+
intosql = IntoSQL::Client.new(api_key: ENV.fetch("INTOSQL_API_KEY"))
|
|
17
|
+
|
|
18
|
+
conversion = intosql.convert("ventas.xlsx", spec: IntoSQL.spec(dialect: "postgresql", table: "ventas"))
|
|
19
|
+
|
|
20
|
+
puts conversion.report[:rows][:converted], conversion.output.url
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
The file streams to storage as it is; the output URL downloads the script for ten minutes. Read the conversion again (`intosql.conversions.retrieve(id)`) for a fresh one.
|
|
24
|
+
|
|
25
|
+
## Convert a file by URL
|
|
26
|
+
|
|
27
|
+
```ruby
|
|
28
|
+
conversion = intosql.convert_url(
|
|
29
|
+
"https://example.com/exports/ventas.csv",
|
|
30
|
+
spec: IntoSQL.spec(dialect: "mysql")
|
|
31
|
+
)
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
## Check a webhook
|
|
35
|
+
|
|
36
|
+
Save a webhook address in Settings, API keys, and IntoSQL calls it when a conversion is done, failed or canceled.
|
|
37
|
+
|
|
38
|
+
```ruby
|
|
39
|
+
# Raises IntoSQL::SignatureError when the signature doesn't match.
|
|
40
|
+
event = IntoSQL.parse_webhook(
|
|
41
|
+
request.raw_post,
|
|
42
|
+
request.headers["IntoSQL-Signature"].to_s,
|
|
43
|
+
ENV.fetch("INTOSQL_WEBHOOK_SECRET")
|
|
44
|
+
)
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
## Errors and retries
|
|
48
|
+
|
|
49
|
+
Failed requests raise `IntoSQL::APIError` with the HTTP `status`, a `code` that doesn't change (`needs_pro`, `over_quota`, `too_big`, `invalid_spec`, `invalid_url` and more) and the `request_id`. A conversion that ends as failed raises `IntoSQL::ConversionFailedError` with its `code` and the engine's `problems`. Rate limits, a full account, server errors and lost connections are retried twice with the same idempotency key (`max_retries:` changes it); `already_started` never is.
|
|
50
|
+
|
|
51
|
+
Every endpoint: [intosql.com/developers](https://intosql.com/developers).
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require "securerandom"
|
|
5
|
+
require "uri"
|
|
6
|
+
|
|
7
|
+
module IntoSQL
|
|
8
|
+
# The IntoSQL API: convert spreadsheets, CSV and JSON into SQL on IntoSQL's machines.
|
|
9
|
+
class Client
|
|
10
|
+
DEFAULT_BASE_URL = "https://intosql.com/api/v1"
|
|
11
|
+
|
|
12
|
+
# Codes worth sending again: waiting fixes them. Never `already_started`, which means it is on its way.
|
|
13
|
+
RETRYABLE_CODES = %w[rate_limited too_many_active request_in_progress].freeze
|
|
14
|
+
|
|
15
|
+
# Statuses from the network in between (gateways, timeouts), which carry no code of ours.
|
|
16
|
+
RETRYABLE_STATUSES = [408, 425, 502, 503, 504].freeze
|
|
17
|
+
|
|
18
|
+
attr_reader :conversions
|
|
19
|
+
|
|
20
|
+
# `max_retries`: attempts after the first one for rate limits, a full account, server errors
|
|
21
|
+
# and lost connections. `timeout`: seconds per request for the default transport.
|
|
22
|
+
# `transport`, `sleeper` and `clock` replace the network, `sleep` and the monotonic clock.
|
|
23
|
+
def initialize(api_key:, base_url: DEFAULT_BASE_URL, max_retries: 2, timeout: 60, transport: nil,
|
|
24
|
+
sleeper: nil, clock: nil)
|
|
25
|
+
raise ArgumentError, "An API key is required: make one in Settings, API keys." if api_key.to_s.empty?
|
|
26
|
+
|
|
27
|
+
@api_key = api_key
|
|
28
|
+
@base_url = base_url.chomp("/")
|
|
29
|
+
@max_retries = max_retries
|
|
30
|
+
@transport = transport || NetHTTPTransport.new(timeout: timeout)
|
|
31
|
+
@sleeper = sleeper || ->(seconds) { Kernel.sleep(seconds) }
|
|
32
|
+
@clock = clock || -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }
|
|
33
|
+
@conversions = Conversions.new(self)
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# Uploads `file` (a path, or an IO with `bytes:` when it has no `size`), converts it and waits
|
|
37
|
+
# for the result. `name` defaults to the file's base name.
|
|
38
|
+
def convert(file, spec:, name: nil, bytes: nil, timeout: Conversions::WAIT_TIMEOUT, interval: nil)
|
|
39
|
+
opened = Source.open(file, bytes: bytes, name: name) do |source|
|
|
40
|
+
raise ArgumentError, "Pass name: so IntoSQL can tell the file's format." if source.name.nil?
|
|
41
|
+
|
|
42
|
+
conversion = conversions.create(name: source.name, bytes: source.bytes, spec: spec)
|
|
43
|
+
conversions.upload(conversion, source.io, bytes: source.bytes)
|
|
44
|
+
conversion
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
conversions.start(opened.id)
|
|
48
|
+
conversions.wait(opened.id, timeout: timeout, interval: interval)
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
# Converts the file at a public HTTPS address and waits for the result.
|
|
52
|
+
def convert_url(url, spec:, name: nil, timeout: Conversions::WAIT_TIMEOUT, interval: nil)
|
|
53
|
+
queued = conversions.create_from_url(url: url, spec: spec, name: name)
|
|
54
|
+
|
|
55
|
+
conversions.wait(queued.id, timeout: timeout, interval: interval)
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# Bytes of conversions the plan includes this month, used and left.
|
|
59
|
+
def usage
|
|
60
|
+
Usage.from_h(request("GET", "/usage").fetch(:data))
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# @api private
|
|
64
|
+
def request(method, path, body = nil)
|
|
65
|
+
url = resolve(path)
|
|
66
|
+
headers = headers_for(method, body)
|
|
67
|
+
payload = body && JSON.generate(body)
|
|
68
|
+
attempt = 0
|
|
69
|
+
|
|
70
|
+
loop do
|
|
71
|
+
response = send_once(method, url, headers, payload, attempt)
|
|
72
|
+
|
|
73
|
+
if response.nil?
|
|
74
|
+
attempt += 1
|
|
75
|
+
next
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
return parse(response) if response.success?
|
|
79
|
+
|
|
80
|
+
error = to_error(response)
|
|
81
|
+
raise error unless retryable?(error) && attempt < @max_retries
|
|
82
|
+
|
|
83
|
+
pause(backoff(attempt, response.header("retry-after")))
|
|
84
|
+
attempt += 1
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
# @api private
|
|
89
|
+
def transport_call(**request)
|
|
90
|
+
@transport.call(**request)
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
# @api private
|
|
94
|
+
def pause(seconds)
|
|
95
|
+
@sleeper.call(seconds)
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
# @api private
|
|
99
|
+
def now
|
|
100
|
+
@clock.call
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
private
|
|
104
|
+
|
|
105
|
+
# The response, or nil when the connection failed and another attempt is due.
|
|
106
|
+
def send_once(method, url, headers, payload, attempt)
|
|
107
|
+
@transport.call(method: method, url: url, headers: headers, body: payload)
|
|
108
|
+
rescue ConnectionError
|
|
109
|
+
raise if attempt >= @max_retries
|
|
110
|
+
|
|
111
|
+
pause(backoff(attempt, nil))
|
|
112
|
+
nil
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
def headers_for(method, body)
|
|
116
|
+
headers = {
|
|
117
|
+
"Authorization" => "Bearer #{@api_key}",
|
|
118
|
+
"Accept" => "application/json",
|
|
119
|
+
"User-Agent" => "intosql-ruby/#{VERSION}"
|
|
120
|
+
}
|
|
121
|
+
headers["Content-Type"] = "application/json" unless body.nil?
|
|
122
|
+
headers["Idempotency-Key"] = SecureRandom.uuid if method == "POST"
|
|
123
|
+
headers
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
def parse(response)
|
|
127
|
+
response.status == 204 ? nil : JSON.parse(response.body, symbolize_names: true)
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
# A path under the API, or one of its own absolute addresses (pagination links).
|
|
131
|
+
def resolve(path)
|
|
132
|
+
return "#{@base_url}#{path}" unless path.start_with?("http")
|
|
133
|
+
|
|
134
|
+
origin = origin_of(path)
|
|
135
|
+
raise Error, "Refusing to send the API key to #{origin}." unless origin == origin_of(@base_url)
|
|
136
|
+
|
|
137
|
+
path
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
def origin_of(url)
|
|
141
|
+
uri = URI(url)
|
|
142
|
+
port = uri.port == uri.default_port ? "" : ":#{uri.port}"
|
|
143
|
+
|
|
144
|
+
"#{uri.scheme}://#{uri.host}#{port}"
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
def to_error(response)
|
|
148
|
+
error = error_body(response)
|
|
149
|
+
|
|
150
|
+
APIError.new(
|
|
151
|
+
response.status,
|
|
152
|
+
error[:code] || "http_error",
|
|
153
|
+
error[:message] || "The API answered #{response.status}.",
|
|
154
|
+
error[:fields] || {},
|
|
155
|
+
error[:request_id] || response.header("x-request-id")
|
|
156
|
+
)
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
# The answer's `error` object, or an empty hash when it has none.
|
|
160
|
+
def error_body(response)
|
|
161
|
+
body = JSON.parse(response.body, symbolize_names: true)
|
|
162
|
+
body.is_a?(Hash) && body[:error].is_a?(Hash) ? body[:error] : {}
|
|
163
|
+
rescue JSON::ParserError
|
|
164
|
+
{}
|
|
165
|
+
end
|
|
166
|
+
|
|
167
|
+
def retryable?(error)
|
|
168
|
+
RETRYABLE_CODES.include?(error.code) || RETRYABLE_STATUSES.include?(error.status) || error.status >= 500
|
|
169
|
+
end
|
|
170
|
+
|
|
171
|
+
def backoff(attempt, retry_after)
|
|
172
|
+
seconds = Float(retry_after, exception: false)
|
|
173
|
+
|
|
174
|
+
return seconds if seconds&.finite? && seconds.positive?
|
|
175
|
+
|
|
176
|
+
[0.5 * (2**attempt), 8.0].min
|
|
177
|
+
end
|
|
178
|
+
end
|
|
179
|
+
end
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "uri"
|
|
4
|
+
|
|
5
|
+
module IntoSQL
|
|
6
|
+
# Each step of a conversion on its own, for uploading from somewhere else, starting later, or
|
|
7
|
+
# not waiting. Reach it as `client.conversions`.
|
|
8
|
+
class Conversions
|
|
9
|
+
# Seconds `wait` keeps reading before it gives up: one hour.
|
|
10
|
+
WAIT_TIMEOUT = 3600
|
|
11
|
+
|
|
12
|
+
def initialize(client)
|
|
13
|
+
@client = client
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
# Opens a conversion; the answer has the address to upload the file to.
|
|
17
|
+
def create(name:, bytes:, spec:)
|
|
18
|
+
conversion("POST", "/conversions", { name: name, bytes: bytes, spec: spec })
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# Queues a conversion of the file at a public HTTPS address.
|
|
22
|
+
def create_from_url(url:, spec:, name: nil)
|
|
23
|
+
conversion("POST", "/conversions", { url: url, spec: spec, name: name }.compact)
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
# Sends the file to the address `create` answered with, streamed as it is. Not retried: a
|
|
27
|
+
# failed upload raises APIError with code `upload_failed`.
|
|
28
|
+
def upload(conversion, file, bytes: nil)
|
|
29
|
+
target = conversion.upload
|
|
30
|
+
raise Error, "Conversion #{conversion.id} has no upload address: open it with create." if target.nil?
|
|
31
|
+
|
|
32
|
+
Source.open(file, bytes: bytes) { |source| put(target, source) }
|
|
33
|
+
|
|
34
|
+
nil
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# Starts a conversion once its file is uploaded whole.
|
|
38
|
+
def start(id)
|
|
39
|
+
conversion("POST", "#{path(id)}/start")
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def retrieve(id)
|
|
43
|
+
conversion("GET", path(id))
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# Every conversion, newest first, as a lazy Enumerator that fetches a page at a time and
|
|
47
|
+
# follows `links.next`; `status` keeps only those in it.
|
|
48
|
+
def list(limit: nil, status: nil)
|
|
49
|
+
query = URI.encode_www_form({ limit: limit, status: status }.compact)
|
|
50
|
+
first = query.empty? ? "/conversions" : "/conversions?#{query}"
|
|
51
|
+
|
|
52
|
+
Enumerator.new do |yielder|
|
|
53
|
+
address = first
|
|
54
|
+
|
|
55
|
+
until address.nil?
|
|
56
|
+
page = Page.from_h(@client.request("GET", address)) { |item| Conversion.from_h(item) }
|
|
57
|
+
page.data.each { |item| yielder << item }
|
|
58
|
+
address = page.links[:next]
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# Stops a conversion that hasn't ended; canceling it again answers the same.
|
|
64
|
+
def cancel(id)
|
|
65
|
+
conversion("POST", "#{path(id)}/cancel")
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# Sends a conversion that didn't come out right to IntoSQL, so support for it gets added.
|
|
69
|
+
# `consent` must be true: the person agrees to send it. `share: "file"` sends the file too.
|
|
70
|
+
def create_case(id, consent:, share: nil, email: nil, note: nil)
|
|
71
|
+
raise ArgumentError, "consent must be true: the person agrees to send it." unless consent == true
|
|
72
|
+
|
|
73
|
+
body = { consent: true, share: share, email: email, note: note }.compact
|
|
74
|
+
|
|
75
|
+
Case.from_h(@client.request("POST", "#{path(id)}/cases", body).fetch(:data))
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# Deletes its files now instead of after 24 hours.
|
|
79
|
+
def delete(id)
|
|
80
|
+
@client.request("DELETE", path(id))
|
|
81
|
+
|
|
82
|
+
nil
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
# Reads the conversion until it is done; raises ConversionFailedError if it fails,
|
|
86
|
+
# ConversionCanceledError if it is canceled and TimeoutError after `timeout` seconds. Pauses
|
|
87
|
+
# grow from half a second to 5 seconds, or are `interval` seconds when given.
|
|
88
|
+
def wait(id, timeout: WAIT_TIMEOUT, interval: nil)
|
|
89
|
+
deadline = @client.now + timeout
|
|
90
|
+
attempt = 0
|
|
91
|
+
|
|
92
|
+
loop do
|
|
93
|
+
conversion = retrieve(id)
|
|
94
|
+
return conversion if ended?(conversion)
|
|
95
|
+
raise TimeoutError.new(id, conversion.status) if @client.now >= deadline
|
|
96
|
+
|
|
97
|
+
@client.pause(interval || [0.5 * (2**attempt), 5.0].min)
|
|
98
|
+
attempt += 1
|
|
99
|
+
end
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
private
|
|
103
|
+
|
|
104
|
+
# True when done; raises when failed or canceled; false while it is still on its way.
|
|
105
|
+
def ended?(conversion)
|
|
106
|
+
case conversion.status
|
|
107
|
+
when ConversionStatus::DONE then true
|
|
108
|
+
when ConversionStatus::FAILED then raise ConversionFailedError.new(conversion.id, conversion.error)
|
|
109
|
+
when ConversionStatus::CANCELED then raise ConversionCanceledError, conversion.id
|
|
110
|
+
else false
|
|
111
|
+
end
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
def put(target, source)
|
|
115
|
+
response = @client.transport_call(
|
|
116
|
+
method: "PUT", url: target.url, headers: upload_headers(target, source), body: source.io
|
|
117
|
+
)
|
|
118
|
+
return if response.success?
|
|
119
|
+
|
|
120
|
+
raise APIError.new(response.status, "upload_failed", "The upload answered #{response.status}.")
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
def conversion(method, address, body = nil)
|
|
124
|
+
Conversion.from_h(@client.request(method, address, body).fetch(:data))
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
def path(id)
|
|
128
|
+
"/conversions/#{escape(id)}"
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
# The same as JavaScript's encodeURIComponent.
|
|
132
|
+
def escape(id)
|
|
133
|
+
id.to_s.b.gsub(/[^A-Za-z0-9\-_.!~*'()]/n) { |char| format("%%%02X", char.ord) }
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
def upload_headers(target, source)
|
|
137
|
+
headers = target.headers.to_h { |name, value| [name.to_s, Array(value).join(", ")] }
|
|
138
|
+
names = headers.keys.map(&:downcase)
|
|
139
|
+
headers["Content-Length"] = source.bytes.to_s unless names.include?("content-length")
|
|
140
|
+
headers["Content-Type"] = "application/octet-stream" unless names.include?("content-type")
|
|
141
|
+
headers
|
|
142
|
+
end
|
|
143
|
+
end
|
|
144
|
+
end
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module IntoSQL
|
|
4
|
+
# Everything the gem raises on purpose inherits from this class.
|
|
5
|
+
class Error < StandardError; end
|
|
6
|
+
|
|
7
|
+
# An answer from the IntoSQL API that wasn't a success. `code` doesn't change between versions.
|
|
8
|
+
class APIError < Error
|
|
9
|
+
attr_reader :status, :code, :fields, :request_id
|
|
10
|
+
|
|
11
|
+
def initialize(status, code, message, fields = {}, request_id = nil)
|
|
12
|
+
super(message)
|
|
13
|
+
@status = status
|
|
14
|
+
@code = code
|
|
15
|
+
@fields = fields
|
|
16
|
+
@request_id = request_id
|
|
17
|
+
end
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
# A conversion that ended as `failed`, with the stable code and the engine's problems.
|
|
21
|
+
class ConversionFailedError < Error
|
|
22
|
+
attr_reader :id, :code, :problems
|
|
23
|
+
|
|
24
|
+
def initialize(id, failure)
|
|
25
|
+
super(failure&.message || "Conversion #{id} failed.")
|
|
26
|
+
@id = id
|
|
27
|
+
@code = failure&.code || "internal_error"
|
|
28
|
+
@problems = failure&.problems || []
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# A conversion that was canceled while it was awaited.
|
|
33
|
+
class ConversionCanceledError < Error
|
|
34
|
+
attr_reader :id
|
|
35
|
+
|
|
36
|
+
def initialize(id)
|
|
37
|
+
super("Conversion #{id} was canceled.")
|
|
38
|
+
@id = id
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
# Waiting gave up before the conversion ended; it keeps running on IntoSQL's servers.
|
|
43
|
+
class TimeoutError < Error
|
|
44
|
+
attr_reader :id, :status
|
|
45
|
+
|
|
46
|
+
def initialize(id, status)
|
|
47
|
+
super("Conversion #{id} is still #{status}.")
|
|
48
|
+
@id = id
|
|
49
|
+
@status = status
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
# The connection failed (after the retries); `cause` is the network error.
|
|
54
|
+
class ConnectionError < Error; end
|
|
55
|
+
|
|
56
|
+
# A webhook call whose IntoSQL-Signature doesn't match its body.
|
|
57
|
+
class SignatureError < Error; end
|
|
58
|
+
end
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module IntoSQL
|
|
4
|
+
# The statuses a conversion goes through. The API may add new ones; they arrive as plain strings.
|
|
5
|
+
module ConversionStatus
|
|
6
|
+
AWAITING_UPLOAD = "awaiting_upload"
|
|
7
|
+
QUEUED = "queued"
|
|
8
|
+
RUNNING = "running"
|
|
9
|
+
DONE = "done"
|
|
10
|
+
FAILED = "failed"
|
|
11
|
+
CANCELED = "canceled"
|
|
12
|
+
ALL = [AWAITING_UPLOAD, QUEUED, RUNNING, DONE, FAILED, CANCELED].freeze
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
# Builds a struct from an API hash, keeping only the keys it knows.
|
|
16
|
+
module Model
|
|
17
|
+
def from_h(hash)
|
|
18
|
+
new(**hash.slice(*members))
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
# Why a conversion failed. `code` is stable; new codes may be added within v1.
|
|
23
|
+
ConversionFailure = Struct.new(:code, :message, :problems, keyword_init: true) do
|
|
24
|
+
extend Model
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
# A finished conversion's result, downloadable for ten minutes.
|
|
28
|
+
Output = Struct.new(:bytes, :url, keyword_init: true) do
|
|
29
|
+
extend Model
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# Where to send the file, right after `create`: `http_method` (`PUT`), `url` and `headers`.
|
|
33
|
+
UploadTarget = Struct.new(:http_method, :url, :headers, keyword_init: true) do
|
|
34
|
+
def self.from_h(hash)
|
|
35
|
+
new(http_method: hash[:method], url: hash[:url], headers: hash[:headers])
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# A conversion as the API answers it. `spec` and `report` are hashes with symbol keys.
|
|
40
|
+
Conversion = Struct.new(
|
|
41
|
+
:id, :status, :name, :bytes, :spec, :report, :error, :output, :upload,
|
|
42
|
+
:created_at, :started_at, :finished_at, :expires_at,
|
|
43
|
+
keyword_init: true
|
|
44
|
+
) do
|
|
45
|
+
def self.from_h(hash)
|
|
46
|
+
fields = hash.slice(*members)
|
|
47
|
+
fields[:error] &&= ConversionFailure.from_h(fields[:error])
|
|
48
|
+
fields[:output] &&= Output.from_h(fields[:output])
|
|
49
|
+
fields[:upload] &&= UploadTarget.from_h(fields[:upload])
|
|
50
|
+
new(**fields)
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# A conversion sent to IntoSQL because it didn't come out right.
|
|
55
|
+
Case = Struct.new(:id, :conversion_id, :share, :status, :created_at, :expires_at, keyword_init: true) do
|
|
56
|
+
extend Model
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# Bytes of conversions the plan includes this month, used and left.
|
|
60
|
+
Usage = Struct.new(:limit, :used, :left, :period_start, keyword_init: true) do
|
|
61
|
+
extend Model
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
# One page of a list: `data`, `links` (`next`, `prev`) and `meta`.
|
|
65
|
+
Page = Struct.new(:data, :links, :meta, keyword_init: true) do
|
|
66
|
+
def self.from_h(hash, &item)
|
|
67
|
+
new(data: hash.fetch(:data).map(&item), links: hash.fetch(:links), meta: hash[:meta])
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
# What a webhook call carries. `id` is the same on every retry: handle each event once.
|
|
72
|
+
# `data` is a Conversion for `conversion.*` events and a hash otherwise (`ping`: `{ sent_at: }`).
|
|
73
|
+
WebhookEvent = Struct.new(:id, :type, :created_at, :data, keyword_init: true) do
|
|
74
|
+
def self.from_h(hash)
|
|
75
|
+
data = hash[:data]
|
|
76
|
+
data = Conversion.from_h(data) if hash[:type].to_s.start_with?("conversion.") && data.is_a?(Hash)
|
|
77
|
+
new(**hash.slice(*members), data: data)
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
end
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "pathname"
|
|
4
|
+
|
|
5
|
+
module IntoSQL
|
|
6
|
+
# A file to upload: a path (String or Pathname), opened and closed here, or an IO that is
|
|
7
|
+
# streamed as it is. An IO without `size` (a pipe, a socket) needs `bytes:`.
|
|
8
|
+
# @api private
|
|
9
|
+
Source = Struct.new(:io, :bytes, :name, keyword_init: true) do
|
|
10
|
+
def self.open(file, bytes: nil, name: nil, &)
|
|
11
|
+
return from_path(file, bytes, name, &) if file.is_a?(String) || file.is_a?(Pathname)
|
|
12
|
+
return yield from_io(file, bytes, name) if file.respond_to?(:read)
|
|
13
|
+
|
|
14
|
+
raise ArgumentError, "Upload a path or an IO, not #{file.class}."
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def self.from_path(path, bytes, name)
|
|
18
|
+
File.open(path, "rb") do |io|
|
|
19
|
+
yield new(io: io, bytes: bytes || io.size, name: name || File.basename(path))
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def self.from_io(io, bytes, name)
|
|
24
|
+
bytes ||= io.size if io.respond_to?(:size)
|
|
25
|
+
raise ArgumentError, "Pass bytes: with the exact size of this #{io.class}." if bytes.nil?
|
|
26
|
+
|
|
27
|
+
path = io.path if io.respond_to?(:path)
|
|
28
|
+
name ||= File.basename(path) if path
|
|
29
|
+
new(io: io, bytes: bytes, name: name)
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
data/lib/intosql/spec.rb
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# Builds conversion specs.
|
|
4
|
+
module IntoSQL
|
|
5
|
+
# A conversion request with IntoSQL's defaults: types inferred, names in snake_case, batched
|
|
6
|
+
# inserts. Returns a hash with symbol keys; change it before sending for anything else.
|
|
7
|
+
def self.spec(dialect:, format: "sql", table: nil, schema: nil, sheet: nil, create_table: true, batch_rows: 500)
|
|
8
|
+
{
|
|
9
|
+
version: 1,
|
|
10
|
+
input: { sheet: sheet, header_row: nil, delimiter: nil, encoding: nil },
|
|
11
|
+
output: {
|
|
12
|
+
format: format, dialect: dialect, table: table, schema: schema, create_table: create_table,
|
|
13
|
+
column_names: "snake_case", style: "inserts", batch_rows: batch_rows
|
|
14
|
+
},
|
|
15
|
+
inference: { sample_rows: 10_000, date_order: "auto", keep_leading_zeros: true, locale: nil },
|
|
16
|
+
columns: []
|
|
17
|
+
}
|
|
18
|
+
end
|
|
19
|
+
end
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "net/http"
|
|
4
|
+
require "openssl"
|
|
5
|
+
require "uri"
|
|
6
|
+
|
|
7
|
+
module IntoSQL
|
|
8
|
+
# What a transport answers: the status, headers with lowercase names, and the body as a string.
|
|
9
|
+
Response = Struct.new(:status, :headers, :body, keyword_init: true) do
|
|
10
|
+
def success?
|
|
11
|
+
status.between?(200, 299)
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def header(name)
|
|
15
|
+
headers[name.downcase]
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
# The default transport, on `net/http`. A transport is anything with
|
|
20
|
+
# `call(method:, url:, headers:, body:)` that returns a Response and raises ConnectionError when
|
|
21
|
+
# the connection fails. `body` is nil, a String, or an IO to stream.
|
|
22
|
+
class NetHTTPTransport
|
|
23
|
+
REQUESTS = {
|
|
24
|
+
"GET" => Net::HTTP::Get,
|
|
25
|
+
"POST" => Net::HTTP::Post,
|
|
26
|
+
"PUT" => Net::HTTP::Put,
|
|
27
|
+
"DELETE" => Net::HTTP::Delete
|
|
28
|
+
}.freeze
|
|
29
|
+
|
|
30
|
+
NETWORK_ERRORS = [
|
|
31
|
+
IOError, SystemCallError, SocketError, Timeout::Error, OpenSSL::SSL::SSLError, Net::ProtocolError
|
|
32
|
+
].freeze
|
|
33
|
+
|
|
34
|
+
# `timeout` is in seconds, for connecting, sending and reading each answer.
|
|
35
|
+
def initialize(timeout: 60)
|
|
36
|
+
@timeout = timeout
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def call(method:, url:, headers:, body: nil)
|
|
40
|
+
uri = URI(url)
|
|
41
|
+
request = REQUESTS.fetch(method).new(uri, headers)
|
|
42
|
+
attach(request, body)
|
|
43
|
+
to_response(connect(uri) { |http| http.request(request) })
|
|
44
|
+
rescue *NETWORK_ERRORS => e
|
|
45
|
+
raise ConnectionError, "#{method} #{uri.host} failed: #{e.message}"
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
private
|
|
49
|
+
|
|
50
|
+
def to_response(response)
|
|
51
|
+
Response.new(
|
|
52
|
+
status: response.code.to_i,
|
|
53
|
+
headers: response.to_hash.transform_values { |values| values.join(", ") },
|
|
54
|
+
body: response.body.to_s
|
|
55
|
+
)
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def attach(request, body)
|
|
59
|
+
if body.respond_to?(:read)
|
|
60
|
+
request.body_stream = body
|
|
61
|
+
elsif body
|
|
62
|
+
request.body = body
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def connect(uri, &)
|
|
67
|
+
Net::HTTP.start(
|
|
68
|
+
uri.host, uri.port,
|
|
69
|
+
use_ssl: uri.scheme == "https",
|
|
70
|
+
open_timeout: @timeout, read_timeout: @timeout, write_timeout: @timeout,
|
|
71
|
+
&
|
|
72
|
+
)
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
end
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require "openssl"
|
|
5
|
+
|
|
6
|
+
# Checks and reads webhook calls.
|
|
7
|
+
module IntoSQL
|
|
8
|
+
# Whether a webhook call came from IntoSQL: `header` is the `IntoSQL-Signature` header, `body`
|
|
9
|
+
# the raw request body, `secret` the signing secret from Settings, API keys. `now` (Unix
|
|
10
|
+
# seconds) fixes the current time in tests.
|
|
11
|
+
def self.verify_webhook(body, header, secret, tolerance: 300, now: nil)
|
|
12
|
+
time, signature = Webhook.signature_parts(header)
|
|
13
|
+
now ||= Time.now.to_i
|
|
14
|
+
|
|
15
|
+
return false if time.nil? || signature.nil? || (now - time).abs > tolerance
|
|
16
|
+
|
|
17
|
+
expected = OpenSSL::HMAC.hexdigest("SHA256", secret, "#{time}.".b + body.to_s.b)
|
|
18
|
+
|
|
19
|
+
expected.bytesize == signature.bytesize && OpenSSL.fixed_length_secure_compare(expected, signature)
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
# The event of a webhook call, once its signature checks out. Raises SignatureError when it
|
|
23
|
+
# doesn't, so a forged or replayed call never reaches your code.
|
|
24
|
+
def self.parse_webhook(body, header, secret, tolerance: 300, now: nil)
|
|
25
|
+
unless verify_webhook(body, header, secret, tolerance: tolerance, now: now)
|
|
26
|
+
raise SignatureError, "The IntoSQL-Signature does not match this body."
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
WebhookEvent.from_h(JSON.parse(body, symbolize_names: true))
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# @api private
|
|
33
|
+
module Webhook
|
|
34
|
+
# The timestamp and signature of a `t=TIME,v1=SIGNATURE` header, nil for each one missing.
|
|
35
|
+
def self.signature_parts(header)
|
|
36
|
+
parts = header.to_s.split(",").to_h do |part|
|
|
37
|
+
key, value = part.strip.split("=", -1)
|
|
38
|
+
[key, value]
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
[Integer(parts["t"].to_s, 10, exception: false), parts["v1"]]
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
end
|
data/lib/intosql.rb
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "intosql/version"
|
|
4
|
+
require_relative "intosql/errors"
|
|
5
|
+
require_relative "intosql/models"
|
|
6
|
+
require_relative "intosql/transport"
|
|
7
|
+
require_relative "intosql/source"
|
|
8
|
+
require_relative "intosql/spec"
|
|
9
|
+
require_relative "intosql/webhooks"
|
|
10
|
+
require_relative "intosql/conversions"
|
|
11
|
+
require_relative "intosql/client"
|
|
12
|
+
|
|
13
|
+
# Convert spreadsheets, CSV and JSON into SQL with the IntoSQL API.
|
|
14
|
+
module IntoSQL
|
|
15
|
+
end
|
metadata
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
|
2
|
+
name: intosql
|
|
3
|
+
version: !ruby/object:Gem::Version
|
|
4
|
+
version: 0.1.0
|
|
5
|
+
platform: ruby
|
|
6
|
+
authors:
|
|
7
|
+
- IntoSQL
|
|
8
|
+
bindir: bin
|
|
9
|
+
cert_chain: []
|
|
10
|
+
date: 1980-01-02 00:00:00.000000000 Z
|
|
11
|
+
dependencies: []
|
|
12
|
+
description: Upload XLSX, XLS, ODS, CSV or JSON files and get SQL for PostgreSQL,
|
|
13
|
+
MySQL, MariaDB, SQLite, SQL Server or Oracle back from the IntoSQL API, with webhook
|
|
14
|
+
verification.
|
|
15
|
+
executables: []
|
|
16
|
+
extensions: []
|
|
17
|
+
extra_rdoc_files: []
|
|
18
|
+
files:
|
|
19
|
+
- CHANGELOG.md
|
|
20
|
+
- LICENSE
|
|
21
|
+
- README.md
|
|
22
|
+
- lib/intosql.rb
|
|
23
|
+
- lib/intosql/client.rb
|
|
24
|
+
- lib/intosql/conversions.rb
|
|
25
|
+
- lib/intosql/errors.rb
|
|
26
|
+
- lib/intosql/models.rb
|
|
27
|
+
- lib/intosql/source.rb
|
|
28
|
+
- lib/intosql/spec.rb
|
|
29
|
+
- lib/intosql/transport.rb
|
|
30
|
+
- lib/intosql/version.rb
|
|
31
|
+
- lib/intosql/webhooks.rb
|
|
32
|
+
homepage: https://intosql.com/docs/sdks/ruby
|
|
33
|
+
licenses:
|
|
34
|
+
- MIT
|
|
35
|
+
metadata:
|
|
36
|
+
homepage_uri: https://intosql.com/docs/sdks/ruby
|
|
37
|
+
source_code_uri: https://github.com/alfonsobries/intosql/tree/main/sdks/ruby
|
|
38
|
+
changelog_uri: https://github.com/alfonsobries/intosql/blob/main/sdks/ruby/CHANGELOG.md
|
|
39
|
+
documentation_uri: https://intosql.com/docs/sdks/ruby
|
|
40
|
+
rubygems_mfa_required: 'false'
|
|
41
|
+
rdoc_options: []
|
|
42
|
+
require_paths:
|
|
43
|
+
- lib
|
|
44
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
|
45
|
+
requirements:
|
|
46
|
+
- - ">="
|
|
47
|
+
- !ruby/object:Gem::Version
|
|
48
|
+
version: '3.1'
|
|
49
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
50
|
+
requirements:
|
|
51
|
+
- - ">="
|
|
52
|
+
- !ruby/object:Gem::Version
|
|
53
|
+
version: '0'
|
|
54
|
+
requirements: []
|
|
55
|
+
rubygems_version: 3.6.9
|
|
56
|
+
specification_version: 4
|
|
57
|
+
summary: Convert spreadsheets, CSV and JSON into SQL with the IntoSQL API.
|
|
58
|
+
test_files: []
|