itsi 0.1.14 → 0.1.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (169) hide show
  1. checksums.yaml +4 -4
  2. data/Cargo.lock +126 -272
  3. data/Cargo.toml +6 -0
  4. data/crates/itsi_error/Cargo.toml +1 -0
  5. data/crates/itsi_error/src/lib.rs +100 -10
  6. data/crates/itsi_scheduler/src/itsi_scheduler.rs +1 -1
  7. data/crates/itsi_server/Cargo.toml +12 -11
  8. data/crates/itsi_server/src/default_responses/html/401.html +68 -0
  9. data/crates/itsi_server/src/default_responses/html/403.html +68 -0
  10. data/crates/itsi_server/src/default_responses/html/404.html +68 -0
  11. data/crates/itsi_server/src/default_responses/html/413.html +71 -0
  12. data/crates/itsi_server/src/default_responses/html/429.html +68 -0
  13. data/crates/itsi_server/src/default_responses/html/500.html +71 -0
  14. data/crates/itsi_server/src/default_responses/html/502.html +71 -0
  15. data/crates/itsi_server/src/default_responses/html/503.html +68 -0
  16. data/crates/itsi_server/src/default_responses/html/504.html +69 -0
  17. data/crates/itsi_server/src/default_responses/html/index.html +238 -0
  18. data/crates/itsi_server/src/default_responses/json/401.json +6 -0
  19. data/crates/itsi_server/src/default_responses/json/403.json +6 -0
  20. data/crates/itsi_server/src/default_responses/json/404.json +6 -0
  21. data/crates/itsi_server/src/default_responses/json/413.json +6 -0
  22. data/crates/itsi_server/src/default_responses/json/429.json +6 -0
  23. data/crates/itsi_server/src/default_responses/json/500.json +6 -0
  24. data/crates/itsi_server/src/default_responses/json/502.json +6 -0
  25. data/crates/itsi_server/src/default_responses/json/503.json +6 -0
  26. data/crates/itsi_server/src/default_responses/json/504.json +6 -0
  27. data/crates/itsi_server/src/default_responses/mod.rs +11 -0
  28. data/crates/itsi_server/src/lib.rs +58 -26
  29. data/crates/itsi_server/src/prelude.rs +2 -0
  30. data/crates/itsi_server/src/ruby_types/README.md +21 -0
  31. data/crates/itsi_server/src/ruby_types/itsi_body_proxy/mod.rs +8 -6
  32. data/crates/itsi_server/src/ruby_types/itsi_grpc_call.rs +344 -0
  33. data/crates/itsi_server/src/ruby_types/{itsi_grpc_stream → itsi_grpc_response_stream}/mod.rs +121 -73
  34. data/crates/itsi_server/src/ruby_types/itsi_http_request.rs +103 -40
  35. data/crates/itsi_server/src/ruby_types/itsi_http_response.rs +8 -5
  36. data/crates/itsi_server/src/ruby_types/itsi_server/file_watcher.rs +4 -4
  37. data/crates/itsi_server/src/ruby_types/itsi_server/itsi_server_config.rs +37 -17
  38. data/crates/itsi_server/src/ruby_types/itsi_server.rs +4 -3
  39. data/crates/itsi_server/src/ruby_types/mod.rs +6 -13
  40. data/crates/itsi_server/src/server/{bind.rs → binds/bind.rs} +23 -4
  41. data/crates/itsi_server/src/server/{listener.rs → binds/listener.rs} +24 -10
  42. data/crates/itsi_server/src/server/binds/mod.rs +4 -0
  43. data/crates/itsi_server/src/server/{tls.rs → binds/tls.rs} +9 -4
  44. data/crates/itsi_server/src/server/http_message_types.rs +97 -0
  45. data/crates/itsi_server/src/server/io_stream.rs +2 -1
  46. data/crates/itsi_server/src/server/middleware_stack/middleware.rs +28 -16
  47. data/crates/itsi_server/src/server/middleware_stack/middlewares/allow_list.rs +17 -8
  48. data/crates/itsi_server/src/server/middleware_stack/middlewares/auth_api_key.rs +47 -18
  49. data/crates/itsi_server/src/server/middleware_stack/middlewares/auth_basic.rs +13 -9
  50. data/crates/itsi_server/src/server/middleware_stack/middlewares/auth_jwt.rs +50 -29
  51. data/crates/itsi_server/src/server/middleware_stack/middlewares/cache_control.rs +5 -2
  52. data/crates/itsi_server/src/server/middleware_stack/middlewares/compression.rs +37 -48
  53. data/crates/itsi_server/src/server/middleware_stack/middlewares/cors.rs +25 -20
  54. data/crates/itsi_server/src/server/middleware_stack/middlewares/deny_list.rs +14 -7
  55. data/crates/itsi_server/src/server/middleware_stack/middlewares/error_response/default_responses.rs +190 -0
  56. data/crates/itsi_server/src/server/middleware_stack/middlewares/error_response.rs +125 -95
  57. data/crates/itsi_server/src/server/middleware_stack/middlewares/etag.rs +9 -5
  58. data/crates/itsi_server/src/server/middleware_stack/middlewares/header_interpretation.rs +1 -4
  59. data/crates/itsi_server/src/server/middleware_stack/middlewares/intrusion_protection.rs +25 -19
  60. data/crates/itsi_server/src/server/middleware_stack/middlewares/log_requests.rs +4 -4
  61. data/crates/itsi_server/src/server/middleware_stack/middlewares/max_body.rs +47 -0
  62. data/crates/itsi_server/src/server/middleware_stack/middlewares/mod.rs +9 -4
  63. data/crates/itsi_server/src/server/middleware_stack/middlewares/proxy.rs +260 -62
  64. data/crates/itsi_server/src/server/middleware_stack/middlewares/rate_limit.rs +29 -22
  65. data/crates/itsi_server/src/server/middleware_stack/middlewares/redirect.rs +6 -6
  66. data/crates/itsi_server/src/server/middleware_stack/middlewares/request_headers.rs +6 -5
  67. data/crates/itsi_server/src/server/middleware_stack/middlewares/response_headers.rs +4 -2
  68. data/crates/itsi_server/src/server/middleware_stack/middlewares/ruby_app.rs +51 -18
  69. data/crates/itsi_server/src/server/middleware_stack/middlewares/static_assets.rs +31 -13
  70. data/crates/itsi_server/src/server/middleware_stack/middlewares/static_response.rs +55 -0
  71. data/crates/itsi_server/src/server/middleware_stack/middlewares/string_rewrite.rs +13 -8
  72. data/crates/itsi_server/src/server/middleware_stack/mod.rs +101 -69
  73. data/crates/itsi_server/src/server/mod.rs +3 -9
  74. data/crates/itsi_server/src/server/process_worker.rs +21 -3
  75. data/crates/itsi_server/src/server/request_job.rs +2 -2
  76. data/crates/itsi_server/src/server/serve_strategy/cluster_mode.rs +8 -3
  77. data/crates/itsi_server/src/server/serve_strategy/single_mode.rs +26 -26
  78. data/crates/itsi_server/src/server/signal.rs +24 -41
  79. data/crates/itsi_server/src/server/size_limited_incoming.rs +101 -0
  80. data/crates/itsi_server/src/server/thread_worker.rs +59 -28
  81. data/crates/itsi_server/src/services/itsi_http_service.rs +239 -0
  82. data/crates/itsi_server/src/services/mime_types.rs +1416 -0
  83. data/crates/itsi_server/src/services/mod.rs +6 -0
  84. data/crates/itsi_server/src/services/password_hasher.rs +83 -0
  85. data/crates/itsi_server/src/{server → services}/rate_limiter.rs +35 -31
  86. data/crates/itsi_server/src/{server → services}/static_file_server.rs +521 -181
  87. data/crates/itsi_tracing/src/lib.rs +145 -55
  88. data/{Itsi.rb → foo/Itsi.rb} +6 -9
  89. data/gems/scheduler/Cargo.lock +7 -0
  90. data/gems/scheduler/lib/itsi/scheduler/version.rb +1 -1
  91. data/gems/scheduler/test/helpers/test_helper.rb +0 -1
  92. data/gems/scheduler/test/test_address_resolve.rb +0 -1
  93. data/gems/scheduler/test/test_network_io.rb +1 -1
  94. data/gems/scheduler/test/test_process_wait.rb +0 -1
  95. data/gems/server/Cargo.lock +126 -272
  96. data/gems/server/exe/itsi +65 -19
  97. data/gems/server/itsi-server.gemspec +4 -3
  98. data/gems/server/lib/itsi/http_request/response_status_shortcodes.rb +74 -0
  99. data/gems/server/lib/itsi/http_request.rb +117 -17
  100. data/gems/server/lib/itsi/http_response.rb +2 -0
  101. data/gems/server/lib/itsi/passfile.rb +109 -0
  102. data/gems/server/lib/itsi/server/config/dsl.rb +171 -99
  103. data/gems/server/lib/itsi/server/config.rb +58 -23
  104. data/gems/server/lib/itsi/server/default_app/default_app.rb +25 -29
  105. data/gems/server/lib/itsi/server/default_app/index.html +113 -89
  106. data/gems/server/lib/itsi/server/{Itsi.rb → default_config/Itsi-rackup.rb} +1 -1
  107. data/gems/server/lib/itsi/server/default_config/Itsi.rb +107 -0
  108. data/gems/server/lib/itsi/server/grpc/grpc_call.rb +246 -0
  109. data/gems/server/lib/itsi/server/grpc/grpc_interface.rb +100 -0
  110. data/gems/server/lib/itsi/server/grpc/reflection/v1/reflection_pb.rb +26 -0
  111. data/gems/server/lib/itsi/server/grpc/reflection/v1/reflection_services_pb.rb +122 -0
  112. data/gems/server/lib/itsi/server/route_tester.rb +107 -0
  113. data/gems/server/lib/itsi/server/typed_handlers/param_parser.rb +200 -0
  114. data/gems/server/lib/itsi/server/typed_handlers/source_parser.rb +55 -0
  115. data/gems/server/lib/itsi/server/typed_handlers.rb +17 -0
  116. data/gems/server/lib/itsi/server/version.rb +1 -1
  117. data/gems/server/lib/itsi/server.rb +82 -12
  118. data/gems/server/lib/ruby_lsp/itsi/addon.rb +111 -0
  119. data/gems/server/lib/shell_completions/completions.rb +26 -0
  120. data/gems/server/test/helpers/test_helper.rb +2 -1
  121. data/lib/itsi/version.rb +1 -1
  122. data/sandbox/README.md +5 -0
  123. data/sandbox/itsi_file/Gemfile +4 -2
  124. data/sandbox/itsi_file/Gemfile.lock +48 -6
  125. data/sandbox/itsi_file/Itsi.rb +327 -129
  126. data/sandbox/itsi_file/call.json +1 -0
  127. data/sandbox/itsi_file/echo_client/Gemfile +10 -0
  128. data/sandbox/itsi_file/echo_client/Gemfile.lock +27 -0
  129. data/sandbox/itsi_file/echo_client/README.md +95 -0
  130. data/sandbox/itsi_file/echo_client/echo_client.rb +164 -0
  131. data/sandbox/itsi_file/echo_client/gen_proto.sh +17 -0
  132. data/sandbox/itsi_file/echo_client/lib/echo_pb.rb +16 -0
  133. data/sandbox/itsi_file/echo_client/lib/echo_services_pb.rb +29 -0
  134. data/sandbox/itsi_file/echo_client/run_client.rb +64 -0
  135. data/sandbox/itsi_file/echo_client/test_compressions.sh +20 -0
  136. data/sandbox/itsi_file/echo_service_nonitsi/Gemfile +10 -0
  137. data/sandbox/itsi_file/echo_service_nonitsi/Gemfile.lock +79 -0
  138. data/sandbox/itsi_file/echo_service_nonitsi/echo.proto +26 -0
  139. data/sandbox/itsi_file/echo_service_nonitsi/echo_pb.rb +16 -0
  140. data/sandbox/itsi_file/echo_service_nonitsi/echo_services_pb.rb +29 -0
  141. data/sandbox/itsi_file/echo_service_nonitsi/server.rb +52 -0
  142. data/sandbox/itsi_sandbox_async/config.ru +0 -1
  143. data/sandbox/itsi_sandbox_rack/Gemfile.lock +2 -2
  144. data/sandbox/itsi_sandbox_rails/Gemfile +2 -2
  145. data/sandbox/itsi_sandbox_rails/Gemfile.lock +76 -2
  146. data/sandbox/itsi_sandbox_rails/app/controllers/home_controller.rb +15 -0
  147. data/sandbox/itsi_sandbox_rails/config/environments/development.rb +1 -0
  148. data/sandbox/itsi_sandbox_rails/config/environments/production.rb +1 -0
  149. data/sandbox/itsi_sandbox_rails/config/routes.rb +2 -0
  150. data/sandbox/itsi_sinatra/app.rb +0 -1
  151. data/sandbox/static_files/.env +1 -0
  152. data/sandbox/static_files/404.html +25 -0
  153. data/sandbox/static_files/_DSC0102.NEF.jpg +0 -0
  154. data/sandbox/static_files/about.html +68 -0
  155. data/sandbox/static_files/tiny.html +1 -0
  156. data/sandbox/static_files/writebook.zip +0 -0
  157. data/tasks.txt +28 -33
  158. metadata +87 -26
  159. data/crates/itsi_error/src/from.rs +0 -68
  160. data/crates/itsi_server/src/ruby_types/itsi_grpc_request.rs +0 -147
  161. data/crates/itsi_server/src/ruby_types/itsi_grpc_response.rs +0 -19
  162. data/crates/itsi_server/src/server/itsi_service.rs +0 -172
  163. data/crates/itsi_server/src/server/middleware_stack/middlewares/grpc_service.rs +0 -72
  164. data/crates/itsi_server/src/server/types.rs +0 -43
  165. data/gems/server/lib/itsi/server/grpc_interface.rb +0 -213
  166. data/sandbox/itsi_file/public/assets/index.html +0 -1
  167. /data/crates/itsi_server/src/server/{bind_protocol.rs → binds/bind_protocol.rs} +0 -0
  168. /data/crates/itsi_server/src/server/{tls → binds/tls}/locked_dir_cache.rs +0 -0
  169. /data/crates/itsi_server/src/{server → services}/cache_store.rs +0 -0
@@ -12,10 +12,12 @@ use std::sync::{
12
12
  };
13
13
  use tokio_stream::StreamExt;
14
14
 
15
+ use crate::server::size_limited_incoming::SizeLimitedIncoming;
16
+
15
17
  #[magnus::wrap(class = "Itsi::BodyProxy", free_immediately, size)]
16
18
  #[derive(Debug, Clone)]
17
19
  pub struct ItsiBodyProxy {
18
- pub incoming: Arc<Mutex<BodyDataStream<Incoming>>>,
20
+ pub incoming: Arc<Mutex<BodyDataStream<SizeLimitedIncoming<Incoming>>>>,
19
21
  pub closed: Arc<AtomicBool>,
20
22
  pub buf: Arc<Mutex<Vec<u8>>>,
21
23
  }
@@ -34,7 +36,7 @@ impl ItsiBody {
34
36
  }
35
37
  }
36
38
  impl ItsiBodyProxy {
37
- pub fn new(incoming: Incoming) -> Self {
39
+ pub fn new(incoming: SizeLimitedIncoming<Incoming>) -> Self {
38
40
  ItsiBodyProxy {
39
41
  incoming: Arc::new(Mutex::new(incoming.into_data_stream())),
40
42
  closed: Arc::new(AtomicBool::new(false)),
@@ -50,7 +52,7 @@ impl ItsiBodyProxy {
50
52
  if let Some(chunk) = block_on(stream.next()) {
51
53
  let chunk = chunk.map_err(|err| {
52
54
  magnus::Error::new(
53
- magnus::exception::exception(),
55
+ magnus::exception::standard_error(),
54
56
  format!("Error reading body {:?}", err),
55
57
  )
56
58
  })?;
@@ -82,7 +84,7 @@ impl ItsiBodyProxy {
82
84
  if let Some(chunk) = block_on(stream.next()) {
83
85
  let chunk = chunk.map_err(|err| {
84
86
  magnus::Error::new(
85
- magnus::exception::exception(),
87
+ magnus::exception::standard_error(),
86
88
  format!("Error reading body {:?}", err),
87
89
  )
88
90
  })?;
@@ -107,7 +109,7 @@ impl ItsiBodyProxy {
107
109
  while let Some(chunk) = block_on(stream.next()) {
108
110
  let chunk = chunk.map_err(|err| {
109
111
  magnus::Error::new(
110
- magnus::exception::exception(),
112
+ magnus::exception::standard_error(),
111
113
  format!("Error reading body {:?}", err),
112
114
  )
113
115
  })?;
@@ -129,7 +131,7 @@ impl ItsiBodyProxy {
129
131
  fn verify_open(&self) -> MagnusResult<()> {
130
132
  if self.closed.load(atomic::Ordering::SeqCst) {
131
133
  return Err(magnus::Error::new(
132
- magnus::exception::exception(),
134
+ magnus::exception::standard_error(),
133
135
  "Body stream is closed",
134
136
  ));
135
137
  }
@@ -0,0 +1,344 @@
1
+ use super::itsi_grpc_response_stream::ItsiGrpcResponseStream;
2
+ use crate::prelude::*;
3
+ use crate::server::http_message_types::{HttpRequest, HttpResponse};
4
+ use crate::server::{byte_frame::ByteFrame, request_job::RequestJob};
5
+ use crate::services::itsi_http_service::HttpRequestContext;
6
+ use async_compression::futures::bufread::{GzipDecoder, GzipEncoder, ZlibDecoder, ZlibEncoder};
7
+ use bytes::Bytes;
8
+ use derive_more::Debug;
9
+ use futures::{executor::block_on, io::Cursor, AsyncReadExt};
10
+ use http::{request::Parts, Response, StatusCode};
11
+ use http_body_util::{combinators::BoxBody, BodyExt, Empty};
12
+ use itsi_error::CLIENT_CONNECTION_CLOSED;
13
+ use itsi_rb_helpers::{print_rb_backtrace, HeapValue};
14
+ use itsi_tracing::debug;
15
+ use magnus::{
16
+ block::Proc,
17
+ error::{ErrorType, Result as MagnusResult},
18
+ Error, Symbol,
19
+ };
20
+ use magnus::{
21
+ value::{LazyId, ReprValue},
22
+ Ruby, Value,
23
+ };
24
+ use regex::Regex;
25
+ use std::sync::LazyLock;
26
+ use std::{collections::HashMap, sync::Arc, time::Instant};
27
+ use tokio::sync::mpsc::{self};
28
+
29
+ static ID_MESSAGE: LazyId = LazyId::new("message");
30
+ static MIN_GZIP_SIZE: u32 = 128;
31
+ static MIN_DEFLATE_SIZE: u32 = 128;
32
+ static METHOD_NAME_REGEX: LazyLock<Regex> =
33
+ LazyLock::new(|| Regex::new(r"([a-z])([A-Z])").expect("Failed to compile regex"));
34
+
35
+ #[derive(Debug)]
36
+ #[magnus::wrap(class = "Itsi::GrpcCall", free_immediately, size)]
37
+ pub struct ItsiGrpcCall {
38
+ pub parts: Parts,
39
+ pub start: Instant,
40
+ pub compression_in: CompressionAlgorithm,
41
+ pub compression_out: CompressionAlgorithm,
42
+ #[debug(skip)]
43
+ pub context: HttpRequestContext,
44
+ #[debug(skip)]
45
+ pub stream: ItsiGrpcResponseStream,
46
+ }
47
+
48
+ #[derive(Debug, Clone)]
49
+ pub enum CompressionAlgorithm {
50
+ None,
51
+ Deflate,
52
+ Gzip,
53
+ }
54
+
55
+ impl ItsiGrpcCall {
56
+ pub fn service_name(&self) -> MagnusResult<String> {
57
+ let path = self.parts.uri.path();
58
+ Ok(path.split('/').nth_back(1).unwrap().to_string())
59
+ }
60
+
61
+ pub fn method_name(&self) -> MagnusResult<Symbol> {
62
+ let path = self.parts.uri.path();
63
+ let method_name = path.split('/').nth_back(0).unwrap();
64
+ let snake_case_method_name = METHOD_NAME_REGEX
65
+ .replace_all(method_name, "${1}_${2}")
66
+ .to_lowercase();
67
+ Ok(Symbol::new(snake_case_method_name))
68
+ }
69
+
70
+ pub fn stream(&self) -> MagnusResult<ItsiGrpcResponseStream> {
71
+ Ok(self.stream.clone())
72
+ }
73
+
74
+ pub fn timeout(&self) -> MagnusResult<Option<f64>> {
75
+ let timeout_str = self
76
+ .parts
77
+ .headers
78
+ .get("grpc-timeout")
79
+ .and_then(|hv| hv.to_str().ok())
80
+ .unwrap_or("");
81
+ Ok(parse_grpc_timeout(timeout_str).ok())
82
+ }
83
+
84
+ pub fn is_cancelled(&self) -> MagnusResult<bool> {
85
+ self.stream.is_cancelled()
86
+ }
87
+
88
+ pub fn add_headers(&self, headers: HashMap<Bytes, Vec<Bytes>>) -> MagnusResult<()> {
89
+ self.stream.add_headers(headers)
90
+ }
91
+
92
+ pub fn content_type_str(&self) -> &str {
93
+ self.parts
94
+ .headers
95
+ .get("Content-Type")
96
+ .and_then(|hv| hv.to_str().ok())
97
+ .unwrap_or("application/x-www-form-urlencoded")
98
+ }
99
+
100
+ pub fn is_json(&self) -> bool {
101
+ self.content_type_str() == "application/json"
102
+ }
103
+
104
+ pub fn process(self, ruby: &Ruby, app_proc: Arc<HeapValue<Proc>>) -> magnus::error::Result<()> {
105
+ let response = self.stream.clone();
106
+ let result = app_proc.call::<_, Value>((self,));
107
+ if let Err(err) = result {
108
+ Self::internal_error(ruby, response, err);
109
+ }
110
+ Ok(())
111
+ }
112
+
113
+ pub fn internal_error(_ruby: &Ruby, stream: ItsiGrpcResponseStream, err: Error) {
114
+ if let Some(rb_err) = err.value() {
115
+ print_rb_backtrace(rb_err);
116
+ stream.internal_server_error(err.to_string());
117
+ } else {
118
+ stream.internal_server_error(err.to_string());
119
+ }
120
+ }
121
+
122
+ pub(crate) async fn process_request(
123
+ app: Arc<HeapValue<Proc>>,
124
+ hyper_request: HttpRequest,
125
+ context: &HttpRequestContext,
126
+ nonblocking: bool,
127
+ ) -> itsi_error::Result<HttpResponse> {
128
+ let (request, mut receiver) = ItsiGrpcCall::new(hyper_request, context).await;
129
+ let shutdown_channel = context.service.shutdown_channel.clone();
130
+ let response_stream = request.stream.clone();
131
+ let sender = if nonblocking {
132
+ &context.nonblocking_sender
133
+ } else {
134
+ &context.sender
135
+ };
136
+ match sender
137
+ .send(RequestJob::ProcessGrpcRequest(request, app))
138
+ .await
139
+ {
140
+ Err(err) => {
141
+ error!("Error occurred: {}", err);
142
+ let mut response = Response::new(BoxBody::new(Empty::new()));
143
+ *response.status_mut() = StatusCode::BAD_REQUEST;
144
+ Ok(response)
145
+ }
146
+ _ => match receiver.recv().await {
147
+ Some(first_frame) => Ok(response_stream
148
+ .build_response(first_frame, receiver, shutdown_channel)
149
+ .await),
150
+ None => Ok(Response::new(BoxBody::new(Empty::new()))),
151
+ },
152
+ }
153
+ }
154
+
155
+ pub fn is_connection_closed_err(ruby: &Ruby, err: &Error) -> bool {
156
+ match err.error_type() {
157
+ ErrorType::Jump(_) => false,
158
+ ErrorType::Error(_, _) => false,
159
+ ErrorType::Exception(exception) => {
160
+ exception.is_kind_of(ruby.exception_eof_error())
161
+ && err
162
+ .value()
163
+ .map(|v| {
164
+ v.funcall::<_, _, String>(*ID_MESSAGE, ())
165
+ .unwrap_or("".to_string())
166
+ .eq(CLIENT_CONNECTION_CLOSED)
167
+ })
168
+ .unwrap_or(false)
169
+ }
170
+ }
171
+ }
172
+
173
+ pub fn should_compress_output(&self, message_size: u32) -> bool {
174
+ match self.compression_out {
175
+ CompressionAlgorithm::Gzip => message_size > MIN_GZIP_SIZE,
176
+ CompressionAlgorithm::Deflate => message_size > MIN_DEFLATE_SIZE,
177
+ CompressionAlgorithm::None => false,
178
+ }
179
+ }
180
+
181
+ pub fn compress_output(&self, bytes: Bytes) -> MagnusResult<Bytes> {
182
+ match self.compression_out {
183
+ CompressionAlgorithm::Gzip => Self::compress_gzip(bytes),
184
+ CompressionAlgorithm::Deflate => Self::compress_deflate(bytes),
185
+ CompressionAlgorithm::None => Ok(bytes),
186
+ }
187
+ }
188
+
189
+ pub fn decompress_input(&self, bytes: Bytes) -> MagnusResult<Bytes> {
190
+ match self.compression_in {
191
+ CompressionAlgorithm::Gzip => Self::decompress_gzip(bytes),
192
+ CompressionAlgorithm::Deflate => Self::decompress_deflate(bytes),
193
+ CompressionAlgorithm::None => Ok(bytes),
194
+ }
195
+ }
196
+
197
+ fn decompress_deflate(input: Bytes) -> MagnusResult<Bytes> {
198
+ let cursor = Cursor::new(input);
199
+ let mut decoder = ZlibDecoder::new(cursor);
200
+
201
+ let result = block_on(async {
202
+ let mut output = Vec::new();
203
+ decoder.read_to_end(&mut output).await?;
204
+ Ok(Bytes::from(output))
205
+ })
206
+ .map_err(|e: std::io::Error| {
207
+ Error::new(
208
+ magnus::exception::standard_error(),
209
+ format!("deflate decompression failed: {}", e),
210
+ )
211
+ })?;
212
+
213
+ Ok(result)
214
+ }
215
+
216
+ fn decompress_gzip(input: Bytes) -> MagnusResult<Bytes> {
217
+ let cursor = Cursor::new(input);
218
+ let mut decoder = GzipDecoder::new(cursor);
219
+
220
+ let result = block_on(async {
221
+ let mut output = Vec::new();
222
+ decoder.read_to_end(&mut output).await?;
223
+ Ok(Bytes::from(output))
224
+ })
225
+ .map_err(|e: std::io::Error| {
226
+ Error::new(
227
+ magnus::exception::standard_error(),
228
+ format!("gzip decompression failed: {}", e),
229
+ )
230
+ })?;
231
+
232
+ Ok(result)
233
+ }
234
+
235
+ fn compress_gzip(input: Bytes) -> MagnusResult<Bytes> {
236
+ let mut output = Vec::with_capacity(input.len() / 2);
237
+ let cursor = Cursor::new(input);
238
+ let mut encoder = GzipEncoder::new(cursor);
239
+
240
+ let result = block_on(async {
241
+ encoder.read_to_end(&mut output).await?;
242
+ Ok::<Bytes, std::io::Error>(output.into())
243
+ })
244
+ .map_err(|e| {
245
+ Error::new(
246
+ magnus::exception::standard_error(),
247
+ format!("gzip compression failed: {e}"),
248
+ )
249
+ })?;
250
+
251
+ Ok(result)
252
+ }
253
+
254
+ fn compress_deflate(input: Bytes) -> MagnusResult<Bytes> {
255
+ let mut output = Vec::with_capacity(input.len() / 2);
256
+ let cursor = Cursor::new(input);
257
+ let mut encoder = ZlibEncoder::new(cursor);
258
+
259
+ let result = block_on(async {
260
+ encoder.read_to_end(&mut output).await?;
261
+ Ok::<Bytes, std::io::Error>(output.into())
262
+ })
263
+ .map_err(|e| {
264
+ Error::new(
265
+ magnus::exception::standard_error(),
266
+ format!("deflate compression failed: {e}"),
267
+ )
268
+ })?;
269
+
270
+ Ok(result)
271
+ }
272
+
273
+ pub(crate) async fn new(
274
+ request: HttpRequest,
275
+ context: &HttpRequestContext,
276
+ ) -> (ItsiGrpcCall, mpsc::Receiver<ByteFrame>) {
277
+ let (parts, body) = request.into_parts();
278
+ let response_channel = mpsc::channel::<ByteFrame>(100);
279
+ let compression_in: CompressionAlgorithm = match parts.headers.get("grpc-encoding") {
280
+ Some(encoding) => match encoding.to_str() {
281
+ Ok(encoding) => match encoding {
282
+ "gzip" => CompressionAlgorithm::Gzip,
283
+ "deflate" => CompressionAlgorithm::Deflate,
284
+ _ => CompressionAlgorithm::None,
285
+ },
286
+ Err(_) => CompressionAlgorithm::None,
287
+ },
288
+ None => CompressionAlgorithm::None,
289
+ };
290
+ let compression_out: CompressionAlgorithm = match parts.headers.get("grpc-accept-encoding")
291
+ {
292
+ Some(accept_encoding) => match accept_encoding.to_str() {
293
+ Ok(accept_encoding) => {
294
+ let encodings: Vec<&str> =
295
+ accept_encoding.split(',').map(|s| s.trim()).collect();
296
+ if encodings.contains(&"gzip") {
297
+ CompressionAlgorithm::Gzip
298
+ } else if encodings.contains(&"deflate") {
299
+ CompressionAlgorithm::Deflate
300
+ } else {
301
+ CompressionAlgorithm::None
302
+ }
303
+ }
304
+ Err(_) => CompressionAlgorithm::None,
305
+ },
306
+ None => CompressionAlgorithm::None,
307
+ };
308
+ (
309
+ Self {
310
+ context: context.clone(),
311
+ start: Instant::now(),
312
+ compression_out: compression_out.clone(),
313
+ compression_in,
314
+ parts,
315
+ stream: ItsiGrpcResponseStream::new(
316
+ compression_out,
317
+ response_channel.0,
318
+ body.into_data_stream(),
319
+ )
320
+ .await,
321
+ },
322
+ response_channel.1,
323
+ )
324
+ }
325
+ }
326
+
327
+ fn parse_grpc_timeout(timeout_str: &str) -> Result<f64> {
328
+ if timeout_str.len() < 2 {
329
+ return Err("Timeout string too short".into());
330
+ }
331
+ let (value_str, unit) = timeout_str.split_at(timeout_str.len() - 1);
332
+ let value: u64 = value_str.parse().map_err(|_| "Invalid timeout value")?;
333
+ let duration_secs = match unit {
334
+ "n" => value as f64 / 1_000_000_000.0, // nanoseconds
335
+ "u" => value as f64 / 1_000_000.0, // microseconds
336
+ "m" => value as f64 / 1_000.0, // milliseconds
337
+ "S" => value as f64, // seconds
338
+ "M" => value as f64 * 60.0, // minutes
339
+ "H" => value as f64 * 3600.0, // hours
340
+ _ => return Err("Invalid timeout unit".into()),
341
+ };
342
+
343
+ Ok(duration_secs)
344
+ }
@@ -1,78 +1,57 @@
1
- use std::{collections::HashMap, sync::Arc};
2
-
3
- use crate::server::{
4
- byte_frame::ByteFrame, serve_strategy::single_mode::RunningPhase, types::HttpResponse,
5
- };
1
+ use super::itsi_grpc_call::CompressionAlgorithm;
2
+ use crate::prelude::*;
3
+ use crate::server::http_message_types::HttpResponse;
4
+ use crate::server::size_limited_incoming::SizeLimitedIncoming;
5
+ use crate::server::{byte_frame::ByteFrame, serve_strategy::single_mode::RunningPhase};
6
6
  use bytes::Bytes;
7
7
  use derive_more::Debug;
8
- use futures::{executor::block_on, stream::unfold};
8
+ use futures::stream::unfold;
9
+ use http::Version;
9
10
  use http::{
10
- header::{HeaderName, HeaderValue, CONTENT_TYPE},
11
+ header::{HeaderName, HeaderValue},
11
12
  HeaderMap, Response,
12
13
  };
13
14
  use http_body_util::{combinators::BoxBody, BodyDataStream, BodyExt, Empty, Full, StreamBody};
14
15
  use hyper::body::{Frame, Incoming};
15
16
  use magnus::error::Result as MagnusResult;
17
+ use nix::unistd::pipe;
16
18
  use parking_lot::Mutex;
17
- use tokio::sync::{
18
- mpsc::{Receiver, Sender},
19
- oneshot, watch,
19
+ use std::sync::atomic::{AtomicBool, Ordering};
20
+ use std::{
21
+ collections::HashMap,
22
+ os::fd::{AsRawFd, FromRawFd, IntoRawFd, OwnedFd},
23
+ sync::Arc,
24
+ };
25
+ use tokio::{
26
+ spawn,
27
+ sync::{
28
+ mpsc::{self, Sender},
29
+ oneshot, watch,
30
+ },
20
31
  };
21
32
  use tokio_stream::{wrappers::ReceiverStream, StreamExt};
22
- use tracing::{error, info, warn};
23
33
 
24
34
  #[derive(Debug, Clone)]
25
- #[magnus::wrap(class = "Itsi::GrpcStream", free_immediately, size)]
26
- pub struct ItsiGrpcStream {
27
- pub inner: Arc<Mutex<ItsiGrpcStreamInner>>,
35
+ #[magnus::wrap(class = "Itsi::GrpcResponseStream", free_immediately, size)]
36
+ pub struct ItsiGrpcResponseStream {
37
+ pub inner: Arc<Mutex<ItsiGrpcResponseStreamInner>>,
38
+ pub cancelled: Arc<AtomicBool>,
28
39
  }
29
40
 
30
41
  #[derive(Debug)]
31
- pub struct ItsiGrpcStreamInner {
32
- pub body: BodyDataStream<Incoming>,
42
+ pub struct ItsiGrpcResponseStreamInner {
43
+ pub incoming_reader: Option<OwnedFd>,
33
44
  pub buf: Vec<u8>,
34
45
  pub response_sender: Sender<ByteFrame>,
35
46
  pub response: Option<HttpResponse>,
47
+ pub response_headers: HeaderMap,
36
48
  trailer_tx: oneshot::Sender<HeaderMap>,
37
49
  trailer_rx: Option<oneshot::Receiver<HeaderMap>>,
38
50
  }
39
51
 
40
- impl ItsiGrpcStreamInner {
41
- pub fn read(&mut self, bytes: usize) -> MagnusResult<Bytes> {
42
- let stream = &mut self.body;
43
- let buf = &mut self.buf;
44
- let mut result = Vec::with_capacity(bytes);
45
-
46
- info!("Entering read with {:?}. Current buf is {:?}", bytes, buf);
47
-
48
- // First, use any data already in the buffer
49
- if !buf.is_empty() {
50
- let remaining = bytes.min(buf.len());
51
- result.extend_from_slice(&buf[..remaining]);
52
- buf.drain(..remaining);
53
- }
54
-
55
- while result.len() < bytes {
56
- if let Some(chunk) = block_on(stream.next()) {
57
- let chunk = chunk.map_err(|err| {
58
- magnus::Error::new(
59
- magnus::exception::exception(),
60
- format!("Error reading body {:?}", err),
61
- )
62
- })?;
63
- let remaining = bytes - result.len();
64
- if chunk.len() > remaining {
65
- result.extend_from_slice(&chunk[..remaining]);
66
- buf.extend_from_slice(&chunk[remaining..]);
67
- } else {
68
- result.extend_from_slice(&chunk);
69
- }
70
- } else {
71
- break;
72
- }
73
- }
74
-
75
- Ok(result.into())
52
+ impl ItsiGrpcResponseStreamInner {
53
+ pub fn reader(&mut self) -> MagnusResult<i32> {
54
+ Ok(self.incoming_reader.take().unwrap().into_raw_fd())
76
55
  }
77
56
 
78
57
  pub fn write(&mut self, bytes: Bytes) -> MagnusResult<()> {
@@ -80,8 +59,8 @@ impl ItsiGrpcStreamInner {
80
59
  .blocking_send(ByteFrame::Data(bytes))
81
60
  .map_err(|err| {
82
61
  magnus::Error::new(
83
- magnus::exception::exception(),
84
- format!("Error writing body {:?}", err),
62
+ magnus::exception::io_error(),
63
+ format!("Trying to write to closed stream: {:?}", err),
85
64
  )
86
65
  })?;
87
66
  Ok(())
@@ -101,39 +80,97 @@ impl ItsiGrpcStreamInner {
101
80
  let trailer_tx = std::mem::replace(&mut self.trailer_tx, oneshot::channel().0);
102
81
  trailer_tx.send(header_map).map_err(|err| {
103
82
  magnus::Error::new(
104
- magnus::exception::exception(),
83
+ magnus::exception::standard_error(),
105
84
  format!("Error sending trailers {:?}", err),
106
85
  )
107
86
  })?;
108
- self.response_sender
109
- .blocking_send(ByteFrame::Empty)
110
- .map_err(|err| {
111
- magnus::Error::new(
112
- magnus::exception::exception(),
113
- format!("Error flushing {:?}", err),
114
- )
87
+ Ok(())
88
+ }
89
+
90
+ pub fn close(&mut self) -> MagnusResult<()> {
91
+ self.response_sender.blocking_send(ByteFrame::Empty).ok();
92
+ Ok(())
93
+ }
94
+
95
+ pub fn add_headers(&mut self, headers: HashMap<Bytes, Vec<Bytes>>) -> MagnusResult<()> {
96
+ for (name, values) in headers {
97
+ let header_name = HeaderName::from_bytes(&name).map_err(|e| {
98
+ itsi_error::ItsiError::InvalidInput(format!(
99
+ "Invalid header name {:?}: {:?}",
100
+ name, e
101
+ ))
115
102
  })?;
103
+ for value in values {
104
+ let header_value = unsafe { HeaderValue::from_maybe_shared_unchecked(value) };
105
+ self.response_headers.insert(&header_name, header_value);
106
+ }
107
+ }
108
+
116
109
  Ok(())
117
110
  }
118
111
  }
119
112
 
120
- impl ItsiGrpcStream {
121
- pub fn new(response_sender: Sender<ByteFrame>, body: BodyDataStream<Incoming>) -> Self {
113
+ impl ItsiGrpcResponseStream {
114
+ pub async fn new(
115
+ compression_out: CompressionAlgorithm,
116
+ response_sender: Sender<ByteFrame>,
117
+ mut body: BodyDataStream<SizeLimitedIncoming<Incoming>>,
118
+ ) -> Self {
122
119
  let (trailer_tx, trailer_rx) = oneshot::channel::<HeaderMap>();
123
- ItsiGrpcStream {
124
- inner: Arc::new(Mutex::new(ItsiGrpcStreamInner {
125
- body,
120
+ let (pipe_read, pipe_write) = pipe().unwrap();
121
+
122
+ nix::fcntl::fcntl(
123
+ pipe_read.as_raw_fd(),
124
+ nix::fcntl::FcntlArg::F_SETFL(nix::fcntl::OFlag::O_NONBLOCK),
125
+ )
126
+ .unwrap();
127
+
128
+ nix::fcntl::fcntl(
129
+ pipe_write.as_raw_fd(),
130
+ nix::fcntl::FcntlArg::F_SETFL(nix::fcntl::OFlag::O_NONBLOCK),
131
+ )
132
+ .unwrap();
133
+
134
+ let pipe_raw_fd = pipe_write.into_raw_fd();
135
+
136
+ let cancelled = Arc::new(AtomicBool::new(false));
137
+ let cancelled_clone = cancelled.clone();
138
+ spawn(async move {
139
+ use std::io::Write;
140
+ let mut write_end = unsafe { std::fs::File::from_raw_fd(pipe_raw_fd) };
141
+ while let Some(Ok(body)) = body.next().await {
142
+ write_end.write_all(&body).unwrap();
143
+ }
144
+ cancelled_clone.store(true, Ordering::SeqCst);
145
+ });
146
+
147
+ let mut response_headers = HeaderMap::new();
148
+
149
+ match compression_out {
150
+ CompressionAlgorithm::None => (),
151
+ CompressionAlgorithm::Deflate => {
152
+ response_headers.insert("grpc-encoding", "deflate".parse().unwrap());
153
+ }
154
+ CompressionAlgorithm::Gzip => {
155
+ response_headers.insert("grpc-encoding", "gzip".parse().unwrap());
156
+ }
157
+ }
158
+ ItsiGrpcResponseStream {
159
+ inner: Arc::new(Mutex::new(ItsiGrpcResponseStreamInner {
126
160
  buf: Vec::new(),
161
+ response_headers,
162
+ incoming_reader: Some(pipe_read),
127
163
  response_sender,
128
164
  response: Some(Response::new(BoxBody::new(Empty::new()))),
129
165
  trailer_tx,
130
166
  trailer_rx: Some(trailer_rx),
131
167
  })),
168
+ cancelled,
132
169
  }
133
170
  }
134
171
 
135
- pub fn read(&self, bytes: usize) -> MagnusResult<Bytes> {
136
- self.inner.lock().read(bytes)
172
+ pub fn reader(&self) -> MagnusResult<i32> {
173
+ self.inner.lock().reader()
137
174
  }
138
175
 
139
176
  pub fn write(&self, bytes: Bytes) -> MagnusResult<()> {
@@ -144,21 +181,32 @@ impl ItsiGrpcStream {
144
181
  self.inner.lock().flush()
145
182
  }
146
183
 
184
+ pub fn is_cancelled(&self) -> MagnusResult<bool> {
185
+ Ok(self.cancelled.load(Ordering::SeqCst))
186
+ }
187
+
147
188
  pub fn send_trailers(&self, trailers: HashMap<String, String>) -> MagnusResult<()> {
148
189
  self.inner.lock().send_trailers(trailers)
149
190
  }
150
191
 
192
+ pub fn close(&self) -> MagnusResult<()> {
193
+ self.inner.lock().close()
194
+ }
195
+
196
+ pub fn add_headers(&self, headers: HashMap<Bytes, Vec<Bytes>>) -> MagnusResult<()> {
197
+ self.inner.lock().add_headers(headers)
198
+ }
199
+
151
200
  pub async fn build_response(
152
201
  &self,
153
202
  first_frame: ByteFrame,
154
- receiver: Receiver<ByteFrame>,
203
+ receiver: mpsc::Receiver<ByteFrame>,
155
204
  shutdown_rx: watch::Receiver<RunningPhase>,
156
205
  ) -> HttpResponse {
157
206
  let mut response = self.inner.lock().response.take().unwrap();
158
207
  let rx = self.inner.lock().trailer_rx.take().unwrap();
159
- response
160
- .headers_mut()
161
- .append(CONTENT_TYPE, "application/grpc".parse().unwrap());
208
+ *response.version_mut() = Version::HTTP_2;
209
+ *response.headers_mut() = self.inner.lock().response_headers.clone();
162
210
  *response.body_mut() = if matches!(first_frame, ByteFrame::Empty) {
163
211
  BoxBody::new(Empty::new())
164
212
  } else if matches!(first_frame, ByteFrame::End(_)) {