itsi 0.1.14 → 0.1.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Cargo.lock +126 -272
- data/Cargo.toml +6 -0
- data/crates/itsi_error/Cargo.toml +1 -0
- data/crates/itsi_error/src/lib.rs +100 -10
- data/crates/itsi_scheduler/src/itsi_scheduler.rs +1 -1
- data/crates/itsi_server/Cargo.toml +12 -11
- data/crates/itsi_server/src/default_responses/html/401.html +68 -0
- data/crates/itsi_server/src/default_responses/html/403.html +68 -0
- data/crates/itsi_server/src/default_responses/html/404.html +68 -0
- data/crates/itsi_server/src/default_responses/html/413.html +71 -0
- data/crates/itsi_server/src/default_responses/html/429.html +68 -0
- data/crates/itsi_server/src/default_responses/html/500.html +71 -0
- data/crates/itsi_server/src/default_responses/html/502.html +71 -0
- data/crates/itsi_server/src/default_responses/html/503.html +68 -0
- data/crates/itsi_server/src/default_responses/html/504.html +69 -0
- data/crates/itsi_server/src/default_responses/html/index.html +238 -0
- data/crates/itsi_server/src/default_responses/json/401.json +6 -0
- data/crates/itsi_server/src/default_responses/json/403.json +6 -0
- data/crates/itsi_server/src/default_responses/json/404.json +6 -0
- data/crates/itsi_server/src/default_responses/json/413.json +6 -0
- data/crates/itsi_server/src/default_responses/json/429.json +6 -0
- data/crates/itsi_server/src/default_responses/json/500.json +6 -0
- data/crates/itsi_server/src/default_responses/json/502.json +6 -0
- data/crates/itsi_server/src/default_responses/json/503.json +6 -0
- data/crates/itsi_server/src/default_responses/json/504.json +6 -0
- data/crates/itsi_server/src/default_responses/mod.rs +11 -0
- data/crates/itsi_server/src/lib.rs +58 -26
- data/crates/itsi_server/src/prelude.rs +2 -0
- data/crates/itsi_server/src/ruby_types/README.md +21 -0
- data/crates/itsi_server/src/ruby_types/itsi_body_proxy/mod.rs +8 -6
- data/crates/itsi_server/src/ruby_types/itsi_grpc_call.rs +344 -0
- data/crates/itsi_server/src/ruby_types/{itsi_grpc_stream → itsi_grpc_response_stream}/mod.rs +121 -73
- data/crates/itsi_server/src/ruby_types/itsi_http_request.rs +103 -40
- data/crates/itsi_server/src/ruby_types/itsi_http_response.rs +8 -5
- data/crates/itsi_server/src/ruby_types/itsi_server/file_watcher.rs +4 -4
- data/crates/itsi_server/src/ruby_types/itsi_server/itsi_server_config.rs +37 -17
- data/crates/itsi_server/src/ruby_types/itsi_server.rs +4 -3
- data/crates/itsi_server/src/ruby_types/mod.rs +6 -13
- data/crates/itsi_server/src/server/{bind.rs → binds/bind.rs} +23 -4
- data/crates/itsi_server/src/server/{listener.rs → binds/listener.rs} +24 -10
- data/crates/itsi_server/src/server/binds/mod.rs +4 -0
- data/crates/itsi_server/src/server/{tls.rs → binds/tls.rs} +9 -4
- data/crates/itsi_server/src/server/http_message_types.rs +97 -0
- data/crates/itsi_server/src/server/io_stream.rs +2 -1
- data/crates/itsi_server/src/server/middleware_stack/middleware.rs +28 -16
- data/crates/itsi_server/src/server/middleware_stack/middlewares/allow_list.rs +17 -8
- data/crates/itsi_server/src/server/middleware_stack/middlewares/auth_api_key.rs +47 -18
- data/crates/itsi_server/src/server/middleware_stack/middlewares/auth_basic.rs +13 -9
- data/crates/itsi_server/src/server/middleware_stack/middlewares/auth_jwt.rs +50 -29
- data/crates/itsi_server/src/server/middleware_stack/middlewares/cache_control.rs +5 -2
- data/crates/itsi_server/src/server/middleware_stack/middlewares/compression.rs +37 -48
- data/crates/itsi_server/src/server/middleware_stack/middlewares/cors.rs +25 -20
- data/crates/itsi_server/src/server/middleware_stack/middlewares/deny_list.rs +14 -7
- data/crates/itsi_server/src/server/middleware_stack/middlewares/error_response/default_responses.rs +190 -0
- data/crates/itsi_server/src/server/middleware_stack/middlewares/error_response.rs +125 -95
- data/crates/itsi_server/src/server/middleware_stack/middlewares/etag.rs +9 -5
- data/crates/itsi_server/src/server/middleware_stack/middlewares/header_interpretation.rs +1 -4
- data/crates/itsi_server/src/server/middleware_stack/middlewares/intrusion_protection.rs +25 -19
- data/crates/itsi_server/src/server/middleware_stack/middlewares/log_requests.rs +4 -4
- data/crates/itsi_server/src/server/middleware_stack/middlewares/max_body.rs +47 -0
- data/crates/itsi_server/src/server/middleware_stack/middlewares/mod.rs +9 -4
- data/crates/itsi_server/src/server/middleware_stack/middlewares/proxy.rs +260 -62
- data/crates/itsi_server/src/server/middleware_stack/middlewares/rate_limit.rs +29 -22
- data/crates/itsi_server/src/server/middleware_stack/middlewares/redirect.rs +6 -6
- data/crates/itsi_server/src/server/middleware_stack/middlewares/request_headers.rs +6 -5
- data/crates/itsi_server/src/server/middleware_stack/middlewares/response_headers.rs +4 -2
- data/crates/itsi_server/src/server/middleware_stack/middlewares/ruby_app.rs +51 -18
- data/crates/itsi_server/src/server/middleware_stack/middlewares/static_assets.rs +31 -13
- data/crates/itsi_server/src/server/middleware_stack/middlewares/static_response.rs +55 -0
- data/crates/itsi_server/src/server/middleware_stack/middlewares/string_rewrite.rs +13 -8
- data/crates/itsi_server/src/server/middleware_stack/mod.rs +101 -69
- data/crates/itsi_server/src/server/mod.rs +3 -9
- data/crates/itsi_server/src/server/process_worker.rs +21 -3
- data/crates/itsi_server/src/server/request_job.rs +2 -2
- data/crates/itsi_server/src/server/serve_strategy/cluster_mode.rs +8 -3
- data/crates/itsi_server/src/server/serve_strategy/single_mode.rs +26 -26
- data/crates/itsi_server/src/server/signal.rs +24 -41
- data/crates/itsi_server/src/server/size_limited_incoming.rs +101 -0
- data/crates/itsi_server/src/server/thread_worker.rs +59 -28
- data/crates/itsi_server/src/services/itsi_http_service.rs +239 -0
- data/crates/itsi_server/src/services/mime_types.rs +1416 -0
- data/crates/itsi_server/src/services/mod.rs +6 -0
- data/crates/itsi_server/src/services/password_hasher.rs +83 -0
- data/crates/itsi_server/src/{server → services}/rate_limiter.rs +35 -31
- data/crates/itsi_server/src/{server → services}/static_file_server.rs +521 -181
- data/crates/itsi_tracing/src/lib.rs +145 -55
- data/{Itsi.rb → foo/Itsi.rb} +6 -9
- data/gems/scheduler/Cargo.lock +7 -0
- data/gems/scheduler/lib/itsi/scheduler/version.rb +1 -1
- data/gems/scheduler/test/helpers/test_helper.rb +0 -1
- data/gems/scheduler/test/test_address_resolve.rb +0 -1
- data/gems/scheduler/test/test_network_io.rb +1 -1
- data/gems/scheduler/test/test_process_wait.rb +0 -1
- data/gems/server/Cargo.lock +126 -272
- data/gems/server/exe/itsi +65 -19
- data/gems/server/itsi-server.gemspec +4 -3
- data/gems/server/lib/itsi/http_request/response_status_shortcodes.rb +74 -0
- data/gems/server/lib/itsi/http_request.rb +117 -17
- data/gems/server/lib/itsi/http_response.rb +2 -0
- data/gems/server/lib/itsi/passfile.rb +109 -0
- data/gems/server/lib/itsi/server/config/dsl.rb +171 -99
- data/gems/server/lib/itsi/server/config.rb +58 -23
- data/gems/server/lib/itsi/server/default_app/default_app.rb +25 -29
- data/gems/server/lib/itsi/server/default_app/index.html +113 -89
- data/gems/server/lib/itsi/server/{Itsi.rb → default_config/Itsi-rackup.rb} +1 -1
- data/gems/server/lib/itsi/server/default_config/Itsi.rb +107 -0
- data/gems/server/lib/itsi/server/grpc/grpc_call.rb +246 -0
- data/gems/server/lib/itsi/server/grpc/grpc_interface.rb +100 -0
- data/gems/server/lib/itsi/server/grpc/reflection/v1/reflection_pb.rb +26 -0
- data/gems/server/lib/itsi/server/grpc/reflection/v1/reflection_services_pb.rb +122 -0
- data/gems/server/lib/itsi/server/route_tester.rb +107 -0
- data/gems/server/lib/itsi/server/typed_handlers/param_parser.rb +200 -0
- data/gems/server/lib/itsi/server/typed_handlers/source_parser.rb +55 -0
- data/gems/server/lib/itsi/server/typed_handlers.rb +17 -0
- data/gems/server/lib/itsi/server/version.rb +1 -1
- data/gems/server/lib/itsi/server.rb +82 -12
- data/gems/server/lib/ruby_lsp/itsi/addon.rb +111 -0
- data/gems/server/lib/shell_completions/completions.rb +26 -0
- data/gems/server/test/helpers/test_helper.rb +2 -1
- data/lib/itsi/version.rb +1 -1
- data/sandbox/README.md +5 -0
- data/sandbox/itsi_file/Gemfile +4 -2
- data/sandbox/itsi_file/Gemfile.lock +48 -6
- data/sandbox/itsi_file/Itsi.rb +327 -129
- data/sandbox/itsi_file/call.json +1 -0
- data/sandbox/itsi_file/echo_client/Gemfile +10 -0
- data/sandbox/itsi_file/echo_client/Gemfile.lock +27 -0
- data/sandbox/itsi_file/echo_client/README.md +95 -0
- data/sandbox/itsi_file/echo_client/echo_client.rb +164 -0
- data/sandbox/itsi_file/echo_client/gen_proto.sh +17 -0
- data/sandbox/itsi_file/echo_client/lib/echo_pb.rb +16 -0
- data/sandbox/itsi_file/echo_client/lib/echo_services_pb.rb +29 -0
- data/sandbox/itsi_file/echo_client/run_client.rb +64 -0
- data/sandbox/itsi_file/echo_client/test_compressions.sh +20 -0
- data/sandbox/itsi_file/echo_service_nonitsi/Gemfile +10 -0
- data/sandbox/itsi_file/echo_service_nonitsi/Gemfile.lock +79 -0
- data/sandbox/itsi_file/echo_service_nonitsi/echo.proto +26 -0
- data/sandbox/itsi_file/echo_service_nonitsi/echo_pb.rb +16 -0
- data/sandbox/itsi_file/echo_service_nonitsi/echo_services_pb.rb +29 -0
- data/sandbox/itsi_file/echo_service_nonitsi/server.rb +52 -0
- data/sandbox/itsi_sandbox_async/config.ru +0 -1
- data/sandbox/itsi_sandbox_rack/Gemfile.lock +2 -2
- data/sandbox/itsi_sandbox_rails/Gemfile +2 -2
- data/sandbox/itsi_sandbox_rails/Gemfile.lock +76 -2
- data/sandbox/itsi_sandbox_rails/app/controllers/home_controller.rb +15 -0
- data/sandbox/itsi_sandbox_rails/config/environments/development.rb +1 -0
- data/sandbox/itsi_sandbox_rails/config/environments/production.rb +1 -0
- data/sandbox/itsi_sandbox_rails/config/routes.rb +2 -0
- data/sandbox/itsi_sinatra/app.rb +0 -1
- data/sandbox/static_files/.env +1 -0
- data/sandbox/static_files/404.html +25 -0
- data/sandbox/static_files/_DSC0102.NEF.jpg +0 -0
- data/sandbox/static_files/about.html +68 -0
- data/sandbox/static_files/tiny.html +1 -0
- data/sandbox/static_files/writebook.zip +0 -0
- data/tasks.txt +28 -33
- metadata +87 -26
- data/crates/itsi_error/src/from.rs +0 -68
- data/crates/itsi_server/src/ruby_types/itsi_grpc_request.rs +0 -147
- data/crates/itsi_server/src/ruby_types/itsi_grpc_response.rs +0 -19
- data/crates/itsi_server/src/server/itsi_service.rs +0 -172
- data/crates/itsi_server/src/server/middleware_stack/middlewares/grpc_service.rs +0 -72
- data/crates/itsi_server/src/server/types.rs +0 -43
- data/gems/server/lib/itsi/server/grpc_interface.rb +0 -213
- data/sandbox/itsi_file/public/assets/index.html +0 -1
- /data/crates/itsi_server/src/server/{bind_protocol.rs → binds/bind_protocol.rs} +0 -0
- /data/crates/itsi_server/src/server/{tls → binds/tls}/locked_dir_cache.rs +0 -0
- /data/crates/itsi_server/src/{server → services}/cache_store.rs +0 -0
@@ -12,10 +12,12 @@ use std::sync::{
|
|
12
12
|
};
|
13
13
|
use tokio_stream::StreamExt;
|
14
14
|
|
15
|
+
use crate::server::size_limited_incoming::SizeLimitedIncoming;
|
16
|
+
|
15
17
|
#[magnus::wrap(class = "Itsi::BodyProxy", free_immediately, size)]
|
16
18
|
#[derive(Debug, Clone)]
|
17
19
|
pub struct ItsiBodyProxy {
|
18
|
-
pub incoming: Arc<Mutex<BodyDataStream<Incoming
|
20
|
+
pub incoming: Arc<Mutex<BodyDataStream<SizeLimitedIncoming<Incoming>>>>,
|
19
21
|
pub closed: Arc<AtomicBool>,
|
20
22
|
pub buf: Arc<Mutex<Vec<u8>>>,
|
21
23
|
}
|
@@ -34,7 +36,7 @@ impl ItsiBody {
|
|
34
36
|
}
|
35
37
|
}
|
36
38
|
impl ItsiBodyProxy {
|
37
|
-
pub fn new(incoming: Incoming) -> Self {
|
39
|
+
pub fn new(incoming: SizeLimitedIncoming<Incoming>) -> Self {
|
38
40
|
ItsiBodyProxy {
|
39
41
|
incoming: Arc::new(Mutex::new(incoming.into_data_stream())),
|
40
42
|
closed: Arc::new(AtomicBool::new(false)),
|
@@ -50,7 +52,7 @@ impl ItsiBodyProxy {
|
|
50
52
|
if let Some(chunk) = block_on(stream.next()) {
|
51
53
|
let chunk = chunk.map_err(|err| {
|
52
54
|
magnus::Error::new(
|
53
|
-
magnus::exception::
|
55
|
+
magnus::exception::standard_error(),
|
54
56
|
format!("Error reading body {:?}", err),
|
55
57
|
)
|
56
58
|
})?;
|
@@ -82,7 +84,7 @@ impl ItsiBodyProxy {
|
|
82
84
|
if let Some(chunk) = block_on(stream.next()) {
|
83
85
|
let chunk = chunk.map_err(|err| {
|
84
86
|
magnus::Error::new(
|
85
|
-
magnus::exception::
|
87
|
+
magnus::exception::standard_error(),
|
86
88
|
format!("Error reading body {:?}", err),
|
87
89
|
)
|
88
90
|
})?;
|
@@ -107,7 +109,7 @@ impl ItsiBodyProxy {
|
|
107
109
|
while let Some(chunk) = block_on(stream.next()) {
|
108
110
|
let chunk = chunk.map_err(|err| {
|
109
111
|
magnus::Error::new(
|
110
|
-
magnus::exception::
|
112
|
+
magnus::exception::standard_error(),
|
111
113
|
format!("Error reading body {:?}", err),
|
112
114
|
)
|
113
115
|
})?;
|
@@ -129,7 +131,7 @@ impl ItsiBodyProxy {
|
|
129
131
|
fn verify_open(&self) -> MagnusResult<()> {
|
130
132
|
if self.closed.load(atomic::Ordering::SeqCst) {
|
131
133
|
return Err(magnus::Error::new(
|
132
|
-
magnus::exception::
|
134
|
+
magnus::exception::standard_error(),
|
133
135
|
"Body stream is closed",
|
134
136
|
));
|
135
137
|
}
|
@@ -0,0 +1,344 @@
|
|
1
|
+
use super::itsi_grpc_response_stream::ItsiGrpcResponseStream;
|
2
|
+
use crate::prelude::*;
|
3
|
+
use crate::server::http_message_types::{HttpRequest, HttpResponse};
|
4
|
+
use crate::server::{byte_frame::ByteFrame, request_job::RequestJob};
|
5
|
+
use crate::services::itsi_http_service::HttpRequestContext;
|
6
|
+
use async_compression::futures::bufread::{GzipDecoder, GzipEncoder, ZlibDecoder, ZlibEncoder};
|
7
|
+
use bytes::Bytes;
|
8
|
+
use derive_more::Debug;
|
9
|
+
use futures::{executor::block_on, io::Cursor, AsyncReadExt};
|
10
|
+
use http::{request::Parts, Response, StatusCode};
|
11
|
+
use http_body_util::{combinators::BoxBody, BodyExt, Empty};
|
12
|
+
use itsi_error::CLIENT_CONNECTION_CLOSED;
|
13
|
+
use itsi_rb_helpers::{print_rb_backtrace, HeapValue};
|
14
|
+
use itsi_tracing::debug;
|
15
|
+
use magnus::{
|
16
|
+
block::Proc,
|
17
|
+
error::{ErrorType, Result as MagnusResult},
|
18
|
+
Error, Symbol,
|
19
|
+
};
|
20
|
+
use magnus::{
|
21
|
+
value::{LazyId, ReprValue},
|
22
|
+
Ruby, Value,
|
23
|
+
};
|
24
|
+
use regex::Regex;
|
25
|
+
use std::sync::LazyLock;
|
26
|
+
use std::{collections::HashMap, sync::Arc, time::Instant};
|
27
|
+
use tokio::sync::mpsc::{self};
|
28
|
+
|
29
|
+
static ID_MESSAGE: LazyId = LazyId::new("message");
|
30
|
+
static MIN_GZIP_SIZE: u32 = 128;
|
31
|
+
static MIN_DEFLATE_SIZE: u32 = 128;
|
32
|
+
static METHOD_NAME_REGEX: LazyLock<Regex> =
|
33
|
+
LazyLock::new(|| Regex::new(r"([a-z])([A-Z])").expect("Failed to compile regex"));
|
34
|
+
|
35
|
+
#[derive(Debug)]
|
36
|
+
#[magnus::wrap(class = "Itsi::GrpcCall", free_immediately, size)]
|
37
|
+
pub struct ItsiGrpcCall {
|
38
|
+
pub parts: Parts,
|
39
|
+
pub start: Instant,
|
40
|
+
pub compression_in: CompressionAlgorithm,
|
41
|
+
pub compression_out: CompressionAlgorithm,
|
42
|
+
#[debug(skip)]
|
43
|
+
pub context: HttpRequestContext,
|
44
|
+
#[debug(skip)]
|
45
|
+
pub stream: ItsiGrpcResponseStream,
|
46
|
+
}
|
47
|
+
|
48
|
+
#[derive(Debug, Clone)]
|
49
|
+
pub enum CompressionAlgorithm {
|
50
|
+
None,
|
51
|
+
Deflate,
|
52
|
+
Gzip,
|
53
|
+
}
|
54
|
+
|
55
|
+
impl ItsiGrpcCall {
|
56
|
+
pub fn service_name(&self) -> MagnusResult<String> {
|
57
|
+
let path = self.parts.uri.path();
|
58
|
+
Ok(path.split('/').nth_back(1).unwrap().to_string())
|
59
|
+
}
|
60
|
+
|
61
|
+
pub fn method_name(&self) -> MagnusResult<Symbol> {
|
62
|
+
let path = self.parts.uri.path();
|
63
|
+
let method_name = path.split('/').nth_back(0).unwrap();
|
64
|
+
let snake_case_method_name = METHOD_NAME_REGEX
|
65
|
+
.replace_all(method_name, "${1}_${2}")
|
66
|
+
.to_lowercase();
|
67
|
+
Ok(Symbol::new(snake_case_method_name))
|
68
|
+
}
|
69
|
+
|
70
|
+
pub fn stream(&self) -> MagnusResult<ItsiGrpcResponseStream> {
|
71
|
+
Ok(self.stream.clone())
|
72
|
+
}
|
73
|
+
|
74
|
+
pub fn timeout(&self) -> MagnusResult<Option<f64>> {
|
75
|
+
let timeout_str = self
|
76
|
+
.parts
|
77
|
+
.headers
|
78
|
+
.get("grpc-timeout")
|
79
|
+
.and_then(|hv| hv.to_str().ok())
|
80
|
+
.unwrap_or("");
|
81
|
+
Ok(parse_grpc_timeout(timeout_str).ok())
|
82
|
+
}
|
83
|
+
|
84
|
+
pub fn is_cancelled(&self) -> MagnusResult<bool> {
|
85
|
+
self.stream.is_cancelled()
|
86
|
+
}
|
87
|
+
|
88
|
+
pub fn add_headers(&self, headers: HashMap<Bytes, Vec<Bytes>>) -> MagnusResult<()> {
|
89
|
+
self.stream.add_headers(headers)
|
90
|
+
}
|
91
|
+
|
92
|
+
pub fn content_type_str(&self) -> &str {
|
93
|
+
self.parts
|
94
|
+
.headers
|
95
|
+
.get("Content-Type")
|
96
|
+
.and_then(|hv| hv.to_str().ok())
|
97
|
+
.unwrap_or("application/x-www-form-urlencoded")
|
98
|
+
}
|
99
|
+
|
100
|
+
pub fn is_json(&self) -> bool {
|
101
|
+
self.content_type_str() == "application/json"
|
102
|
+
}
|
103
|
+
|
104
|
+
pub fn process(self, ruby: &Ruby, app_proc: Arc<HeapValue<Proc>>) -> magnus::error::Result<()> {
|
105
|
+
let response = self.stream.clone();
|
106
|
+
let result = app_proc.call::<_, Value>((self,));
|
107
|
+
if let Err(err) = result {
|
108
|
+
Self::internal_error(ruby, response, err);
|
109
|
+
}
|
110
|
+
Ok(())
|
111
|
+
}
|
112
|
+
|
113
|
+
pub fn internal_error(_ruby: &Ruby, stream: ItsiGrpcResponseStream, err: Error) {
|
114
|
+
if let Some(rb_err) = err.value() {
|
115
|
+
print_rb_backtrace(rb_err);
|
116
|
+
stream.internal_server_error(err.to_string());
|
117
|
+
} else {
|
118
|
+
stream.internal_server_error(err.to_string());
|
119
|
+
}
|
120
|
+
}
|
121
|
+
|
122
|
+
pub(crate) async fn process_request(
|
123
|
+
app: Arc<HeapValue<Proc>>,
|
124
|
+
hyper_request: HttpRequest,
|
125
|
+
context: &HttpRequestContext,
|
126
|
+
nonblocking: bool,
|
127
|
+
) -> itsi_error::Result<HttpResponse> {
|
128
|
+
let (request, mut receiver) = ItsiGrpcCall::new(hyper_request, context).await;
|
129
|
+
let shutdown_channel = context.service.shutdown_channel.clone();
|
130
|
+
let response_stream = request.stream.clone();
|
131
|
+
let sender = if nonblocking {
|
132
|
+
&context.nonblocking_sender
|
133
|
+
} else {
|
134
|
+
&context.sender
|
135
|
+
};
|
136
|
+
match sender
|
137
|
+
.send(RequestJob::ProcessGrpcRequest(request, app))
|
138
|
+
.await
|
139
|
+
{
|
140
|
+
Err(err) => {
|
141
|
+
error!("Error occurred: {}", err);
|
142
|
+
let mut response = Response::new(BoxBody::new(Empty::new()));
|
143
|
+
*response.status_mut() = StatusCode::BAD_REQUEST;
|
144
|
+
Ok(response)
|
145
|
+
}
|
146
|
+
_ => match receiver.recv().await {
|
147
|
+
Some(first_frame) => Ok(response_stream
|
148
|
+
.build_response(first_frame, receiver, shutdown_channel)
|
149
|
+
.await),
|
150
|
+
None => Ok(Response::new(BoxBody::new(Empty::new()))),
|
151
|
+
},
|
152
|
+
}
|
153
|
+
}
|
154
|
+
|
155
|
+
pub fn is_connection_closed_err(ruby: &Ruby, err: &Error) -> bool {
|
156
|
+
match err.error_type() {
|
157
|
+
ErrorType::Jump(_) => false,
|
158
|
+
ErrorType::Error(_, _) => false,
|
159
|
+
ErrorType::Exception(exception) => {
|
160
|
+
exception.is_kind_of(ruby.exception_eof_error())
|
161
|
+
&& err
|
162
|
+
.value()
|
163
|
+
.map(|v| {
|
164
|
+
v.funcall::<_, _, String>(*ID_MESSAGE, ())
|
165
|
+
.unwrap_or("".to_string())
|
166
|
+
.eq(CLIENT_CONNECTION_CLOSED)
|
167
|
+
})
|
168
|
+
.unwrap_or(false)
|
169
|
+
}
|
170
|
+
}
|
171
|
+
}
|
172
|
+
|
173
|
+
pub fn should_compress_output(&self, message_size: u32) -> bool {
|
174
|
+
match self.compression_out {
|
175
|
+
CompressionAlgorithm::Gzip => message_size > MIN_GZIP_SIZE,
|
176
|
+
CompressionAlgorithm::Deflate => message_size > MIN_DEFLATE_SIZE,
|
177
|
+
CompressionAlgorithm::None => false,
|
178
|
+
}
|
179
|
+
}
|
180
|
+
|
181
|
+
pub fn compress_output(&self, bytes: Bytes) -> MagnusResult<Bytes> {
|
182
|
+
match self.compression_out {
|
183
|
+
CompressionAlgorithm::Gzip => Self::compress_gzip(bytes),
|
184
|
+
CompressionAlgorithm::Deflate => Self::compress_deflate(bytes),
|
185
|
+
CompressionAlgorithm::None => Ok(bytes),
|
186
|
+
}
|
187
|
+
}
|
188
|
+
|
189
|
+
pub fn decompress_input(&self, bytes: Bytes) -> MagnusResult<Bytes> {
|
190
|
+
match self.compression_in {
|
191
|
+
CompressionAlgorithm::Gzip => Self::decompress_gzip(bytes),
|
192
|
+
CompressionAlgorithm::Deflate => Self::decompress_deflate(bytes),
|
193
|
+
CompressionAlgorithm::None => Ok(bytes),
|
194
|
+
}
|
195
|
+
}
|
196
|
+
|
197
|
+
fn decompress_deflate(input: Bytes) -> MagnusResult<Bytes> {
|
198
|
+
let cursor = Cursor::new(input);
|
199
|
+
let mut decoder = ZlibDecoder::new(cursor);
|
200
|
+
|
201
|
+
let result = block_on(async {
|
202
|
+
let mut output = Vec::new();
|
203
|
+
decoder.read_to_end(&mut output).await?;
|
204
|
+
Ok(Bytes::from(output))
|
205
|
+
})
|
206
|
+
.map_err(|e: std::io::Error| {
|
207
|
+
Error::new(
|
208
|
+
magnus::exception::standard_error(),
|
209
|
+
format!("deflate decompression failed: {}", e),
|
210
|
+
)
|
211
|
+
})?;
|
212
|
+
|
213
|
+
Ok(result)
|
214
|
+
}
|
215
|
+
|
216
|
+
fn decompress_gzip(input: Bytes) -> MagnusResult<Bytes> {
|
217
|
+
let cursor = Cursor::new(input);
|
218
|
+
let mut decoder = GzipDecoder::new(cursor);
|
219
|
+
|
220
|
+
let result = block_on(async {
|
221
|
+
let mut output = Vec::new();
|
222
|
+
decoder.read_to_end(&mut output).await?;
|
223
|
+
Ok(Bytes::from(output))
|
224
|
+
})
|
225
|
+
.map_err(|e: std::io::Error| {
|
226
|
+
Error::new(
|
227
|
+
magnus::exception::standard_error(),
|
228
|
+
format!("gzip decompression failed: {}", e),
|
229
|
+
)
|
230
|
+
})?;
|
231
|
+
|
232
|
+
Ok(result)
|
233
|
+
}
|
234
|
+
|
235
|
+
fn compress_gzip(input: Bytes) -> MagnusResult<Bytes> {
|
236
|
+
let mut output = Vec::with_capacity(input.len() / 2);
|
237
|
+
let cursor = Cursor::new(input);
|
238
|
+
let mut encoder = GzipEncoder::new(cursor);
|
239
|
+
|
240
|
+
let result = block_on(async {
|
241
|
+
encoder.read_to_end(&mut output).await?;
|
242
|
+
Ok::<Bytes, std::io::Error>(output.into())
|
243
|
+
})
|
244
|
+
.map_err(|e| {
|
245
|
+
Error::new(
|
246
|
+
magnus::exception::standard_error(),
|
247
|
+
format!("gzip compression failed: {e}"),
|
248
|
+
)
|
249
|
+
})?;
|
250
|
+
|
251
|
+
Ok(result)
|
252
|
+
}
|
253
|
+
|
254
|
+
fn compress_deflate(input: Bytes) -> MagnusResult<Bytes> {
|
255
|
+
let mut output = Vec::with_capacity(input.len() / 2);
|
256
|
+
let cursor = Cursor::new(input);
|
257
|
+
let mut encoder = ZlibEncoder::new(cursor);
|
258
|
+
|
259
|
+
let result = block_on(async {
|
260
|
+
encoder.read_to_end(&mut output).await?;
|
261
|
+
Ok::<Bytes, std::io::Error>(output.into())
|
262
|
+
})
|
263
|
+
.map_err(|e| {
|
264
|
+
Error::new(
|
265
|
+
magnus::exception::standard_error(),
|
266
|
+
format!("deflate compression failed: {e}"),
|
267
|
+
)
|
268
|
+
})?;
|
269
|
+
|
270
|
+
Ok(result)
|
271
|
+
}
|
272
|
+
|
273
|
+
pub(crate) async fn new(
|
274
|
+
request: HttpRequest,
|
275
|
+
context: &HttpRequestContext,
|
276
|
+
) -> (ItsiGrpcCall, mpsc::Receiver<ByteFrame>) {
|
277
|
+
let (parts, body) = request.into_parts();
|
278
|
+
let response_channel = mpsc::channel::<ByteFrame>(100);
|
279
|
+
let compression_in: CompressionAlgorithm = match parts.headers.get("grpc-encoding") {
|
280
|
+
Some(encoding) => match encoding.to_str() {
|
281
|
+
Ok(encoding) => match encoding {
|
282
|
+
"gzip" => CompressionAlgorithm::Gzip,
|
283
|
+
"deflate" => CompressionAlgorithm::Deflate,
|
284
|
+
_ => CompressionAlgorithm::None,
|
285
|
+
},
|
286
|
+
Err(_) => CompressionAlgorithm::None,
|
287
|
+
},
|
288
|
+
None => CompressionAlgorithm::None,
|
289
|
+
};
|
290
|
+
let compression_out: CompressionAlgorithm = match parts.headers.get("grpc-accept-encoding")
|
291
|
+
{
|
292
|
+
Some(accept_encoding) => match accept_encoding.to_str() {
|
293
|
+
Ok(accept_encoding) => {
|
294
|
+
let encodings: Vec<&str> =
|
295
|
+
accept_encoding.split(',').map(|s| s.trim()).collect();
|
296
|
+
if encodings.contains(&"gzip") {
|
297
|
+
CompressionAlgorithm::Gzip
|
298
|
+
} else if encodings.contains(&"deflate") {
|
299
|
+
CompressionAlgorithm::Deflate
|
300
|
+
} else {
|
301
|
+
CompressionAlgorithm::None
|
302
|
+
}
|
303
|
+
}
|
304
|
+
Err(_) => CompressionAlgorithm::None,
|
305
|
+
},
|
306
|
+
None => CompressionAlgorithm::None,
|
307
|
+
};
|
308
|
+
(
|
309
|
+
Self {
|
310
|
+
context: context.clone(),
|
311
|
+
start: Instant::now(),
|
312
|
+
compression_out: compression_out.clone(),
|
313
|
+
compression_in,
|
314
|
+
parts,
|
315
|
+
stream: ItsiGrpcResponseStream::new(
|
316
|
+
compression_out,
|
317
|
+
response_channel.0,
|
318
|
+
body.into_data_stream(),
|
319
|
+
)
|
320
|
+
.await,
|
321
|
+
},
|
322
|
+
response_channel.1,
|
323
|
+
)
|
324
|
+
}
|
325
|
+
}
|
326
|
+
|
327
|
+
fn parse_grpc_timeout(timeout_str: &str) -> Result<f64> {
|
328
|
+
if timeout_str.len() < 2 {
|
329
|
+
return Err("Timeout string too short".into());
|
330
|
+
}
|
331
|
+
let (value_str, unit) = timeout_str.split_at(timeout_str.len() - 1);
|
332
|
+
let value: u64 = value_str.parse().map_err(|_| "Invalid timeout value")?;
|
333
|
+
let duration_secs = match unit {
|
334
|
+
"n" => value as f64 / 1_000_000_000.0, // nanoseconds
|
335
|
+
"u" => value as f64 / 1_000_000.0, // microseconds
|
336
|
+
"m" => value as f64 / 1_000.0, // milliseconds
|
337
|
+
"S" => value as f64, // seconds
|
338
|
+
"M" => value as f64 * 60.0, // minutes
|
339
|
+
"H" => value as f64 * 3600.0, // hours
|
340
|
+
_ => return Err("Invalid timeout unit".into()),
|
341
|
+
};
|
342
|
+
|
343
|
+
Ok(duration_secs)
|
344
|
+
}
|
data/crates/itsi_server/src/ruby_types/{itsi_grpc_stream → itsi_grpc_response_stream}/mod.rs
RENAMED
@@ -1,78 +1,57 @@
|
|
1
|
-
use
|
2
|
-
|
3
|
-
use crate::server::
|
4
|
-
|
5
|
-
};
|
1
|
+
use super::itsi_grpc_call::CompressionAlgorithm;
|
2
|
+
use crate::prelude::*;
|
3
|
+
use crate::server::http_message_types::HttpResponse;
|
4
|
+
use crate::server::size_limited_incoming::SizeLimitedIncoming;
|
5
|
+
use crate::server::{byte_frame::ByteFrame, serve_strategy::single_mode::RunningPhase};
|
6
6
|
use bytes::Bytes;
|
7
7
|
use derive_more::Debug;
|
8
|
-
use futures::
|
8
|
+
use futures::stream::unfold;
|
9
|
+
use http::Version;
|
9
10
|
use http::{
|
10
|
-
header::{HeaderName, HeaderValue
|
11
|
+
header::{HeaderName, HeaderValue},
|
11
12
|
HeaderMap, Response,
|
12
13
|
};
|
13
14
|
use http_body_util::{combinators::BoxBody, BodyDataStream, BodyExt, Empty, Full, StreamBody};
|
14
15
|
use hyper::body::{Frame, Incoming};
|
15
16
|
use magnus::error::Result as MagnusResult;
|
17
|
+
use nix::unistd::pipe;
|
16
18
|
use parking_lot::Mutex;
|
17
|
-
use
|
18
|
-
|
19
|
-
|
19
|
+
use std::sync::atomic::{AtomicBool, Ordering};
|
20
|
+
use std::{
|
21
|
+
collections::HashMap,
|
22
|
+
os::fd::{AsRawFd, FromRawFd, IntoRawFd, OwnedFd},
|
23
|
+
sync::Arc,
|
24
|
+
};
|
25
|
+
use tokio::{
|
26
|
+
spawn,
|
27
|
+
sync::{
|
28
|
+
mpsc::{self, Sender},
|
29
|
+
oneshot, watch,
|
30
|
+
},
|
20
31
|
};
|
21
32
|
use tokio_stream::{wrappers::ReceiverStream, StreamExt};
|
22
|
-
use tracing::{error, info, warn};
|
23
33
|
|
24
34
|
#[derive(Debug, Clone)]
|
25
|
-
#[magnus::wrap(class = "Itsi::
|
26
|
-
pub struct
|
27
|
-
pub inner: Arc<Mutex<
|
35
|
+
#[magnus::wrap(class = "Itsi::GrpcResponseStream", free_immediately, size)]
|
36
|
+
pub struct ItsiGrpcResponseStream {
|
37
|
+
pub inner: Arc<Mutex<ItsiGrpcResponseStreamInner>>,
|
38
|
+
pub cancelled: Arc<AtomicBool>,
|
28
39
|
}
|
29
40
|
|
30
41
|
#[derive(Debug)]
|
31
|
-
pub struct
|
32
|
-
pub
|
42
|
+
pub struct ItsiGrpcResponseStreamInner {
|
43
|
+
pub incoming_reader: Option<OwnedFd>,
|
33
44
|
pub buf: Vec<u8>,
|
34
45
|
pub response_sender: Sender<ByteFrame>,
|
35
46
|
pub response: Option<HttpResponse>,
|
47
|
+
pub response_headers: HeaderMap,
|
36
48
|
trailer_tx: oneshot::Sender<HeaderMap>,
|
37
49
|
trailer_rx: Option<oneshot::Receiver<HeaderMap>>,
|
38
50
|
}
|
39
51
|
|
40
|
-
impl
|
41
|
-
pub fn
|
42
|
-
|
43
|
-
let buf = &mut self.buf;
|
44
|
-
let mut result = Vec::with_capacity(bytes);
|
45
|
-
|
46
|
-
info!("Entering read with {:?}. Current buf is {:?}", bytes, buf);
|
47
|
-
|
48
|
-
// First, use any data already in the buffer
|
49
|
-
if !buf.is_empty() {
|
50
|
-
let remaining = bytes.min(buf.len());
|
51
|
-
result.extend_from_slice(&buf[..remaining]);
|
52
|
-
buf.drain(..remaining);
|
53
|
-
}
|
54
|
-
|
55
|
-
while result.len() < bytes {
|
56
|
-
if let Some(chunk) = block_on(stream.next()) {
|
57
|
-
let chunk = chunk.map_err(|err| {
|
58
|
-
magnus::Error::new(
|
59
|
-
magnus::exception::exception(),
|
60
|
-
format!("Error reading body {:?}", err),
|
61
|
-
)
|
62
|
-
})?;
|
63
|
-
let remaining = bytes - result.len();
|
64
|
-
if chunk.len() > remaining {
|
65
|
-
result.extend_from_slice(&chunk[..remaining]);
|
66
|
-
buf.extend_from_slice(&chunk[remaining..]);
|
67
|
-
} else {
|
68
|
-
result.extend_from_slice(&chunk);
|
69
|
-
}
|
70
|
-
} else {
|
71
|
-
break;
|
72
|
-
}
|
73
|
-
}
|
74
|
-
|
75
|
-
Ok(result.into())
|
52
|
+
impl ItsiGrpcResponseStreamInner {
|
53
|
+
pub fn reader(&mut self) -> MagnusResult<i32> {
|
54
|
+
Ok(self.incoming_reader.take().unwrap().into_raw_fd())
|
76
55
|
}
|
77
56
|
|
78
57
|
pub fn write(&mut self, bytes: Bytes) -> MagnusResult<()> {
|
@@ -80,8 +59,8 @@ impl ItsiGrpcStreamInner {
|
|
80
59
|
.blocking_send(ByteFrame::Data(bytes))
|
81
60
|
.map_err(|err| {
|
82
61
|
magnus::Error::new(
|
83
|
-
magnus::exception::
|
84
|
-
format!("
|
62
|
+
magnus::exception::io_error(),
|
63
|
+
format!("Trying to write to closed stream: {:?}", err),
|
85
64
|
)
|
86
65
|
})?;
|
87
66
|
Ok(())
|
@@ -101,39 +80,97 @@ impl ItsiGrpcStreamInner {
|
|
101
80
|
let trailer_tx = std::mem::replace(&mut self.trailer_tx, oneshot::channel().0);
|
102
81
|
trailer_tx.send(header_map).map_err(|err| {
|
103
82
|
magnus::Error::new(
|
104
|
-
magnus::exception::
|
83
|
+
magnus::exception::standard_error(),
|
105
84
|
format!("Error sending trailers {:?}", err),
|
106
85
|
)
|
107
86
|
})?;
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
87
|
+
Ok(())
|
88
|
+
}
|
89
|
+
|
90
|
+
pub fn close(&mut self) -> MagnusResult<()> {
|
91
|
+
self.response_sender.blocking_send(ByteFrame::Empty).ok();
|
92
|
+
Ok(())
|
93
|
+
}
|
94
|
+
|
95
|
+
pub fn add_headers(&mut self, headers: HashMap<Bytes, Vec<Bytes>>) -> MagnusResult<()> {
|
96
|
+
for (name, values) in headers {
|
97
|
+
let header_name = HeaderName::from_bytes(&name).map_err(|e| {
|
98
|
+
itsi_error::ItsiError::InvalidInput(format!(
|
99
|
+
"Invalid header name {:?}: {:?}",
|
100
|
+
name, e
|
101
|
+
))
|
115
102
|
})?;
|
103
|
+
for value in values {
|
104
|
+
let header_value = unsafe { HeaderValue::from_maybe_shared_unchecked(value) };
|
105
|
+
self.response_headers.insert(&header_name, header_value);
|
106
|
+
}
|
107
|
+
}
|
108
|
+
|
116
109
|
Ok(())
|
117
110
|
}
|
118
111
|
}
|
119
112
|
|
120
|
-
impl
|
121
|
-
pub fn new(
|
113
|
+
impl ItsiGrpcResponseStream {
|
114
|
+
pub async fn new(
|
115
|
+
compression_out: CompressionAlgorithm,
|
116
|
+
response_sender: Sender<ByteFrame>,
|
117
|
+
mut body: BodyDataStream<SizeLimitedIncoming<Incoming>>,
|
118
|
+
) -> Self {
|
122
119
|
let (trailer_tx, trailer_rx) = oneshot::channel::<HeaderMap>();
|
123
|
-
|
124
|
-
|
125
|
-
|
120
|
+
let (pipe_read, pipe_write) = pipe().unwrap();
|
121
|
+
|
122
|
+
nix::fcntl::fcntl(
|
123
|
+
pipe_read.as_raw_fd(),
|
124
|
+
nix::fcntl::FcntlArg::F_SETFL(nix::fcntl::OFlag::O_NONBLOCK),
|
125
|
+
)
|
126
|
+
.unwrap();
|
127
|
+
|
128
|
+
nix::fcntl::fcntl(
|
129
|
+
pipe_write.as_raw_fd(),
|
130
|
+
nix::fcntl::FcntlArg::F_SETFL(nix::fcntl::OFlag::O_NONBLOCK),
|
131
|
+
)
|
132
|
+
.unwrap();
|
133
|
+
|
134
|
+
let pipe_raw_fd = pipe_write.into_raw_fd();
|
135
|
+
|
136
|
+
let cancelled = Arc::new(AtomicBool::new(false));
|
137
|
+
let cancelled_clone = cancelled.clone();
|
138
|
+
spawn(async move {
|
139
|
+
use std::io::Write;
|
140
|
+
let mut write_end = unsafe { std::fs::File::from_raw_fd(pipe_raw_fd) };
|
141
|
+
while let Some(Ok(body)) = body.next().await {
|
142
|
+
write_end.write_all(&body).unwrap();
|
143
|
+
}
|
144
|
+
cancelled_clone.store(true, Ordering::SeqCst);
|
145
|
+
});
|
146
|
+
|
147
|
+
let mut response_headers = HeaderMap::new();
|
148
|
+
|
149
|
+
match compression_out {
|
150
|
+
CompressionAlgorithm::None => (),
|
151
|
+
CompressionAlgorithm::Deflate => {
|
152
|
+
response_headers.insert("grpc-encoding", "deflate".parse().unwrap());
|
153
|
+
}
|
154
|
+
CompressionAlgorithm::Gzip => {
|
155
|
+
response_headers.insert("grpc-encoding", "gzip".parse().unwrap());
|
156
|
+
}
|
157
|
+
}
|
158
|
+
ItsiGrpcResponseStream {
|
159
|
+
inner: Arc::new(Mutex::new(ItsiGrpcResponseStreamInner {
|
126
160
|
buf: Vec::new(),
|
161
|
+
response_headers,
|
162
|
+
incoming_reader: Some(pipe_read),
|
127
163
|
response_sender,
|
128
164
|
response: Some(Response::new(BoxBody::new(Empty::new()))),
|
129
165
|
trailer_tx,
|
130
166
|
trailer_rx: Some(trailer_rx),
|
131
167
|
})),
|
168
|
+
cancelled,
|
132
169
|
}
|
133
170
|
}
|
134
171
|
|
135
|
-
pub fn
|
136
|
-
self.inner.lock().
|
172
|
+
pub fn reader(&self) -> MagnusResult<i32> {
|
173
|
+
self.inner.lock().reader()
|
137
174
|
}
|
138
175
|
|
139
176
|
pub fn write(&self, bytes: Bytes) -> MagnusResult<()> {
|
@@ -144,21 +181,32 @@ impl ItsiGrpcStream {
|
|
144
181
|
self.inner.lock().flush()
|
145
182
|
}
|
146
183
|
|
184
|
+
pub fn is_cancelled(&self) -> MagnusResult<bool> {
|
185
|
+
Ok(self.cancelled.load(Ordering::SeqCst))
|
186
|
+
}
|
187
|
+
|
147
188
|
pub fn send_trailers(&self, trailers: HashMap<String, String>) -> MagnusResult<()> {
|
148
189
|
self.inner.lock().send_trailers(trailers)
|
149
190
|
}
|
150
191
|
|
192
|
+
pub fn close(&self) -> MagnusResult<()> {
|
193
|
+
self.inner.lock().close()
|
194
|
+
}
|
195
|
+
|
196
|
+
pub fn add_headers(&self, headers: HashMap<Bytes, Vec<Bytes>>) -> MagnusResult<()> {
|
197
|
+
self.inner.lock().add_headers(headers)
|
198
|
+
}
|
199
|
+
|
151
200
|
pub async fn build_response(
|
152
201
|
&self,
|
153
202
|
first_frame: ByteFrame,
|
154
|
-
receiver: Receiver<ByteFrame>,
|
203
|
+
receiver: mpsc::Receiver<ByteFrame>,
|
155
204
|
shutdown_rx: watch::Receiver<RunningPhase>,
|
156
205
|
) -> HttpResponse {
|
157
206
|
let mut response = self.inner.lock().response.take().unwrap();
|
158
207
|
let rx = self.inner.lock().trailer_rx.take().unwrap();
|
159
|
-
response
|
160
|
-
|
161
|
-
.append(CONTENT_TYPE, "application/grpc".parse().unwrap());
|
208
|
+
*response.version_mut() = Version::HTTP_2;
|
209
|
+
*response.headers_mut() = self.inner.lock().response_headers.clone();
|
162
210
|
*response.body_mut() = if matches!(first_frame, ByteFrame::Empty) {
|
163
211
|
BoxBody::new(Empty::new())
|
164
212
|
} else if matches!(first_frame, ByteFrame::End(_)) {
|