hecks 4.0.0 → 6.0.0

data/rust/host/src/server.rs

@@ -1,12 +1,15 @@
 //! HTTP server for AWS Fargate (`HECKS_SERVE_MODE=1`): the axum fallback route below shares
 //! `dispatch_body` with the Lambda custom-runtime path, so neither repeats the other's logic.
 
+use crate::arg_limits;
 use crate::dispatch;
 use crate::host::Host;
-use crate::journal::LineageConfig;
 use crate::lambda_client::LambdaInvoker;
 use crate::log;
 use crate::rate_limit::{self, RateLimits, Verdict};
+use crate::store::journal_sql::LineageConfig;
+use crate::store::Store;
+use crate::validation;
 use crate::wasm_runner;
 use crate::web;
 use axum::body::Bytes;
@@ -20,26 +23,113 @@ use serde_json::Value;
 use std::net::SocketAddr;
 use std::path::PathBuf;
 use std::sync::Arc;
-use tokio::sync::Mutex;
-use tokio_postgres::Client;
 
 /// The per-request dispatch decision, shared between the Lambda custom-runtime path and the
 /// axum fallback route below. `body` is already-parsed JSON in one of its recognized shapes.
+///
+/// `"bare": true` beside any of them (`verb` with `to`/`with` or `args`, `read`, `query`) answers
+/// the same outcome with each single-field value object as its bare field
+/// (`{"name": "Margherita"}`, not `{"name": {"value": "Margherita"}}`). What is stored is the
+/// same either way; without `bare` the answer is the stored shape.
 pub async fn dispatch_body(body: Value, host: Host<'_>) -> Result<Value, Error> {
+    if let Some(response) = web::render(&body, host).await {
+        return Ok(conceal_internal_error(response));
+    }
+    let answer = kernel_answer(&body, host).await?;
+    if !wants_bare(&body) || wants_preview(&body) {
+        return Ok(answer);
+    }
+    Ok(dispatch::present(host.wasm_path, answer)
+        .await
+        .map_err(|e| format!("{e:#}"))?)
+}
+
+/// A route's `500` keeps its detail in the log and tells the caller only a request id to quote:
+/// an error's text can name tables, queries and paths that are nobody else's business.
+fn conceal_internal_error(mut response: Value) -> Value {
+    if response.get("statusCode").and_then(Value::as_u64) != Some(500) {
+        return response;
+    }
+    let request_id = new_request_id();
+    log::error(
+        "internal_error",
+        serde_json::json!({ "request_id": request_id, "detail": response.get("body") }),
+    );
+    response["headers"]["content-type"] = Value::from("application/json");
+    response["body"] = Value::from(internal_error_body(&request_id).to_string());
+    response["isBase64Encoded"] = Value::Bool(false);
+    response
+}
+
+fn internal_error_body(request_id: &str) -> Value {
+    serde_json::json!({ "error": "internal error", "request_id": request_id })
+}
+
+fn new_request_id() -> String {
+    uuid::Uuid::new_v4().simple().to_string()
+}
+
+/// The `429` a Lambda Function-URL event over a public route's rate limit gets, judged as `admit`
+/// judges a request in serve mode, with `requestContext.http.sourceIp` as the peer. An event with
+/// no route (an internal verb or read invocation) is never limited.
+pub fn lambda_rate_limited(limits: &RateLimits, event: &Value) -> Option<Value> {
+    let (method, path) = envelope_route(event)?;
+    let peer = event
+        .pointer("/requestContext/http/sourceIp")
+        .and_then(Value::as_str)
+        .and_then(|ip| ip.parse().ok());
+    match limits.check(method, path, &event_headers(event), peer) {
+        Verdict::Limited { retry_after_secs } => {
+            Some(rate_limit::too_many_requests_event(retry_after_secs))
+        }
+        Verdict::Allowed => None,
+    }
+}
+
+// A Function-URL event's `headers` object as a `HeaderMap`; a name or value that is not a valid
+// header is left out.
+fn event_headers(event: &Value) -> HeaderMap {
+    let mut headers = HeaderMap::new();
+    for (name, value) in event
+        .get("headers")
+        .and_then(Value::as_object)
+        .into_iter()
+        .flatten()
+    {
+        let parsed = (
+            HeaderName::from_bytes(name.as_bytes()),
+            value.as_str().map(HeaderValue::from_str),
+        );
+        if let (Ok(name), Some(Ok(value))) = parsed {
+            headers.append(name, value);
+        }
+    }
+    headers
+}
+
+fn wants_bare(body: &Value) -> bool {
+    body.get("bare").and_then(Value::as_bool) == Some(true)
+}
+
+// `"dry_run": true` beside a command asks whether it would be accepted, and keeps nothing. Its
+// answer is already the client envelope, so `bare` has nothing to unwrap in it.
+fn wants_preview(body: &Value) -> bool {
+    body.get("dry_run").and_then(Value::as_bool) == Some(true)
+}
+
+// The kernel's outcome for a read, query or command body, in the stored shape.
+async fn kernel_answer(body: &Value, host: Host<'_>) -> Result<Value, Error> {
     let Host {
-        client,
+        store,
         wasm_path,
         config,
         ..
     } = host;
-    if let Some(response) = web::render(&body, host).await {
-        return Ok(response);
-    }
 
     if body.get("read").and_then(serde_json::Value::as_bool) == Some(true) {
         // `{e:#}`: anyhow's alternate Display walks `source()` to the real message,
         // where a bare `tokio_postgres` error's Display is just "db error".
-        let result = dispatch::read(client, wasm_path)
+        let result = dispatch::read(store, wasm_path)
             .await
             .map_err(|e| format!("{e:#}"))?;
         return Ok(result);
@@ -48,10 +138,10 @@ pub async fn dispatch_body(body: Value, host: Host<'_>) -> Result<Value, Error>
     // A query step reads current state and writes nothing to the journal.
     if body.get("query").is_some() {
         return Ok(crate::query_step::answer(
-            &body,
+            body,
             crate::ir::ir(),
             &config.domain,
-            client,
+            store,
             wasm_path,
         )
         .await?);
@@ -77,7 +167,23 @@ pub async fn dispatch_body(body: Value, host: Host<'_>) -> Result<Value, Error>
         actor_id: actor_id.as_deref(),
     };
 
-    let outcome = if let Some(to) = body.get("to").cloned() {
+    let invocation = invocation_of(body)?;
+    if wants_preview(body) {
+        let outcome = dispatch::preview_as(host, &verb, invocation, caller)
+            .await
+            .map_err(|e| format!("{e:#}"))?;
+        return Ok(validation::answer(&outcome.result));
+    }
+    let outcome = dispatch::handle_as(host, &verb, invocation, caller)
+        .await
+        .map_err(|e| format!("{e:#}"))?;
+    Ok(outcome.result)
+}
+
+// The command's arguments in the invocation shape the kernel reads, from whichever of the body's
+// three spellings it used: `to` with `with`, `with` alone, or the legacy `args`.
+fn invocation_of(body: &Value) -> Result<Value, Error> {
+    if let Some(to) = body.get("to").cloned() {
         if body.get("args").is_some() {
             return Err("cannot combine to/with with legacy args".into());
         }
@@ -85,33 +191,25 @@ pub async fn dispatch_body(body: Value, host: Host<'_>) -> Result<Value, Error>
             .get("with")
             .cloned()
             .unwrap_or_else(|| serde_json::json!({}));
-        dispatch::handle_routed_as(host, &verb, to, facts, caller)
-            .await
-            .map_err(|e| format!("{e:#}"))?
-    } else if let Some(facts) = body.get("with").cloned() {
+        return dispatch::routed_invocation(to, facts).map_err(|e| format!("{e:#}").into());
+    }
+    if let Some(facts) = body.get("with").cloned() {
         if body.get("args").is_some() {
             return Err("cannot combine \"with\" with legacy args".into());
         }
-        dispatch::handle_facts_as(host, &verb, facts, caller)
-            .await
-            .map_err(|e| format!("{e:#}"))?
-    } else {
-        let args = body
-            .get("args")
-            .cloned()
-            .unwrap_or_else(|| serde_json::json!({}));
-        dispatch::handle_as(host, &verb, args, caller)
-            .await
-            .map_err(|e| format!("{e:#}"))?
-    };
-    Ok(outcome.result)
+        return dispatch::facts_invocation(facts).map_err(|e| format!("{e:#}").into());
+    }
+    Ok(body
+        .get("args")
+        .cloned()
+        .unwrap_or_else(|| serde_json::json!({})))
 }
 
 /// Everything `dispatch_body` needs, `Arc`-shared for axum's per-request handler cloning.
-/// `client`'s mutex serializes every write onto the one Postgres connection this process holds.
+/// The store serializes every write onto the one journal this process holds.
 #[derive(Clone)]
 pub struct ServerState {
-    pub client: Arc<Mutex<Client>>,
+    pub store: Arc<Store>,
     pub wasm_path: Arc<PathBuf>,
     pub lineage_config: Arc<LineageConfig>,
     pub invoker: Arc<dyn LambdaInvoker>,
@@ -122,7 +220,7 @@ impl ServerState {
     /// The shared pieces a dispatch needs, borrowed for one call.
     pub fn host(&self) -> Host<'_> {
         Host::shared(
-            &self.client,
+            &self.store,
             &self.wasm_path,
             &self.lineage_config,
             &self.invoker,
@@ -254,13 +352,14 @@ async fn route_request(
     match dispatch_body(envelope, state.host()).await {
         Ok(value) => value_to_response(value),
         Err(e) => {
+            let request_id = new_request_id();
             log::error(
                 "dispatch_failed",
-                serde_json::json!({ "error": format!("{e}") }),
+                serde_json::json!({ "error": format!("{e}"), "request_id": request_id }),
             );
             (
                 StatusCode::INTERNAL_SERVER_ERROR,
-                axum::Json(serde_json::json!({ "error": format!("{e}") })),
+                axum::Json(internal_error_body(&request_id)),
             )
                 .into_response()
         }
@@ -306,6 +405,18 @@ fn admit(
     {
         parsed
     } else {
+        if let Some(longest) = arg_limits::oversized(&parsed) {
+            return Err(Box::new(
+                (
+                    StatusCode::PAYLOAD_TOO_LARGE,
+                    format!(
+                        "a string in the body is {longest} bytes; at most {} are accepted",
+                        arg_limits::MAX_STRING_ARG_BYTES
+                    ),
+                )
+                    .into_response(),
+            ));
+        }
         synthesize_function_url_envelope(method, uri, headers, body)
     };
 
@@ -451,6 +562,8 @@ fn value_to_response(value: Value) -> Response {
 #[cfg(test)]
 mod tests {
     use super::*;
+    #[allow(unused_imports)]
+    use crate::store::StoreFixtures;
 
     #[test]
     fn a_function_url_shaped_body_is_recognized_as_internal_dispatch() {
@@ -539,6 +652,28 @@ mod tests {
         .unwrap_or_else(|_| panic!("{from} POST {path} was refused before dispatch"))
     }
 
+    #[test]
+    fn a_public_body_with_a_string_past_the_cap_is_refused_and_a_peers_is_not() {
+        let big = format!(
+            r#"{{"verb":"Post.Publish","with":{{"body":"{}"}}}}"#,
+            "x".repeat(arg_limits::MAX_STRING_ARG_BYTES + 1)
+        );
+        let body = Bytes::from(big.clone());
+        let refused = admit(
+            &limits_from(&[]),
+            peer("203.0.113.7:5000"),
+            &Method::POST,
+            &"/api/posts".parse().unwrap(),
+            &HeaderMap::new(),
+            &body,
+        )
+        .expect_err("a public body past the cap must be refused");
+        assert_eq!(refused.status(), StatusCode::PAYLOAD_TOO_LARGE);
+
+        let envelope = admitted("127.0.0.1:5000", "/anything", &big);
+        assert_eq!(envelope["verb"], "Post.Publish");
+    }
+
     // The gap this closes: a public POST to any path the load balancer forwards carried
     // `{"verb": ..., "role": ...}` or `{"read": true}` straight to the kernel, with the
     // caller's own word for its role and no session check.
@@ -819,6 +954,75 @@ mod tests {
         );
     }
 
+    fn function_url_event(path: &str, source_ip: &str) -> Value {
+        serde_json::json!({
+            "requestContext": { "http": { "method": "POST", "sourceIp": source_ip } },
+            "rawPath": path,
+            "headers": { "content-type": "application/json" },
+            "body": "{}",
+        })
+    }
+
+    #[test]
+    fn a_lambda_burst_from_one_source_ip_is_refused_with_a_429_event() {
+        let limits = limits_from(&[("HECKS_RATE_LIMIT_SUBSCRIBE", "2")]);
+        let event = function_url_event(SUBSCRIBE, "203.0.113.7");
+        assert_eq!(lambda_rate_limited(&limits, &event), None);
+        assert_eq!(lambda_rate_limited(&limits, &event), None);
+        let refused = lambda_rate_limited(&limits, &event).expect("the third is over the limit");
+        assert_eq!(refused["statusCode"], 429);
+        assert!(
+            refused["headers"]["retry-after"].as_str().is_some(),
+            "{refused}"
+        );
+        let body: Value = serde_json::from_str(refused["body"].as_str().unwrap()).unwrap();
+        assert_eq!(body["error"], "too many requests, please try again later");
+    }
+
+    #[test]
+    fn lambda_limits_count_each_source_ip_apart_and_never_an_internal_invocation() {
+        let limits = limits_from(&[("HECKS_RATE_LIMIT_SUBSCRIBE", "1")]);
+        assert_eq!(
+            lambda_rate_limited(&limits, &function_url_event(SUBSCRIBE, "203.0.113.7")),
+            None
+        );
+        assert_eq!(
+            lambda_rate_limited(&limits, &function_url_event(SUBSCRIBE, "203.0.113.8")),
+            None
+        );
+        let read = serde_json::json!({ "read": true });
+        for _ in 0..3 {
+            assert_eq!(lambda_rate_limited(&limits, &read), None);
+        }
+    }
+
+    #[test]
+    fn a_routes_500_reaches_the_caller_as_a_request_id_not_its_detail() {
+        let failed = serde_json::json!({
+            "statusCode": 500,
+            "headers": { "content-type": "text/plain" },
+            "body": "relation \"secret_table\" does not exist",
+            "isBase64Encoded": false,
+        });
+        let concealed = conceal_internal_error(failed);
+        let body: Value = serde_json::from_str(concealed["body"].as_str().unwrap()).unwrap();
+        assert_eq!(body["error"], "internal error");
+        assert_eq!(body["request_id"].as_str().map(str::len), Some(32));
+        assert!(
+            !concealed.to_string().contains("secret_table"),
+            "{concealed}"
+        );
+        assert_eq!(concealed["headers"]["content-type"], "application/json");
+    }
+
+    #[test]
+    fn a_response_that_is_not_a_500_is_passed_through() {
+        for status in [200, 404, 422, 502] {
+            let response = serde_json::json!({ "statusCode": status, "body": "kept" });
+            assert_eq!(conceal_internal_error(response.clone()), response);
+        }
+    }
+
     #[tokio::test]
     async fn registrations_are_limited_apart_from_subscribes() {
         let addr = gated_server(limits_from(&[
@@ -957,8 +1161,8 @@ mod tests {
     // The real service on an ephemeral port, over a scratch Postgres and a cold copy of the
     // checkout fixture's wasm, so the first request is a real compile.
     async fn fixture_host(database: &str, copy_name: &str) -> (SocketAddr, Arc<PathBuf>) {
-        let client = crate::dispatch::tests::scratch_db(database).await;
-        crate::dispatch::tests::provision_lineage(
+        let client = crate::dispatch_tests::tests::scratch_db(database).await;
+        crate::dispatch_tests::tests::provision_lineage(
             &*client.lock().await,
             "CheckoutFixture",
             1,
@@ -967,12 +1171,13 @@ mod tests {
         .await;
         let wasm_path = Arc::new(crate::wasm_runner::tests::cold_copy_of_fixture(copy_name));
         let state = ServerState {
-            client: Arc::new(client),
+            store: Arc::new(client),
             wasm_path: Arc::clone(&wasm_path),
             lineage_config: Arc::new(LineageConfig {
                 domain: "CheckoutFixture".to_string(),
                 era: Some(1),
                 mirrored: None,
+                seal_every: None,
             }),
             invoker: Arc::new(crate::lambda_client::NeverInvoker),
             limits: limits_from(&[]),
@@ -1041,6 +1246,52 @@ mod tests {
         );
     }
 
+    // The checkout fixture's `Event`: `slug`, `name` and `capacity` hold one `value` field and
+    // `price` one `cents` field, so each answers as a bare scalar when asked to.
+    #[tokio::test]
+    async fn a_bare_body_answers_single_field_value_objects_bare_and_the_default_stays_wrapped() {
+        let client = crate::dispatch_tests::tests::memory_store();
+        let config = LineageConfig {
+            domain: "CheckoutFixture".to_string(),
+            era: None,
+            mirrored: None,
+            seal_every: None,
+        };
+        let wasm = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../dist/checkout_fixture.wasm");
+        let host = Host::new(&client, &wasm, &config, &crate::lambda_client::NeverInvoker);
+        let schedule = serde_json::json!({
+            "verb": "CheckoutFixture::Event.Schedule",
+            "role": "Organizer",
+            "with": {"slug": "intro", "name": "Intro", "price": 1500, "capacity": 10},
+            "bare": true,
+        });
+
+        let scheduled = dispatch_body(schedule, host).await.unwrap();
+        let wrapped = dispatch_body(serde_json::json!({"read": true}), host)
+            .await
+            .unwrap();
+        let bare = dispatch_body(serde_json::json!({"read": true, "bare": true}), host)
+            .await
+            .unwrap();
+
+        let event = "CheckoutFixture::Event#intro";
+        assert_eq!(
+            scheduled["instances"][event]["name"], "Intro",
+            "{scheduled}"
+        );
+        assert_eq!(
+            scheduled["events"][0]["payload"]["price"], 1500,
+            "{scheduled}"
+        );
+        assert_eq!(
+            wrapped["instances"][event]["name"],
+            serde_json::json!({"value": "Intro"}),
+            "{wrapped}"
+        );
+        assert_eq!(bare["instances"][event]["price"], 1500, "{bare}");
+        assert_eq!(bare["instances"][event]["capacity"], 10, "{bare}");
+    }
+
     #[test]
     fn build_is_the_env_value_when_set() {
         assert_eq!(version_body("e", "h", Some("cms-1"))["build"], "cms-1");