deepdebug-local-agent 1.4.0 → 1.6.0

package/test/la-protocol-v2/traces.test.js

@@ -1,4 +1,4 @@
-// F3-05b: the 24 whole-session traces of the contract (F3-22), reproduced in Node. "Traces are language
+// F3-05b (+2 of F3-07b, content.index): the 26 whole-session traces of the contract (F3-22), reproduced in Node. "Traces are language
 // neutral; the LA must reproduce each expect.error." Three things are shown for them:
 //   1. the reference CP (test helper) reproduces every trace: same error at the same frame (so the model the
 //      other tests use is the model of the contract);
@@ -18,9 +18,9 @@ import { V2Session } from "../../src/la-protocol-v2/session.js";
 const DIR = path.join(path.dirname(fileURLToPath(import.meta.url)), "../contract/la-protocol-v2/vectors/traces");
 const traces = fs.readdirSync(DIR).filter((f) => f.endsWith(".json")).sort().map((f) => ({ file: f, name: f.replace(".json", ""), ...JSON.parse(fs.readFileSync(path.join(DIR, f), "utf8")) }));
 
-test("the suite has all 24 traces (20 invalid, 4 valid)", () => {
-    assert.equal(traces.length, 24);
-    assert.equal(traces.filter((t) => t.expect.ok).length, 4);
+test("the suite has all 26 traces (21 invalid, 5 valid)", () => {
+    assert.equal(traces.length, 26);
+    assert.equal(traces.filter((t) => t.expect.ok).length, 5);
 });
 
 const textOf = (fr) => {
@@ -223,6 +223,31 @@ test("valid-content-chunks-in-order: the three chunks of the trace go out in ord
     env.session.close();
 });
 
+test("valid-content-index: the two inventories of the trace go out in order after ready and the CP accepts them", async () => {
+    const t = traces.find((x) => x.name === "valid-content-index");
+    const cp = new ReferenceCp({ tenantId: t.session.tenantId, agentId: t.session.agentId });
+    const env = inMemory(cp, { tenantId: t.session.tenantId, agentId: t.session.agentId }).start();
+    await waitFor(() => env.session.state === "ready");
+    for (const fr of LA_FRAMES(t).filter((f) => f.frame.type === "content.index")) {
+        await env.session.send("content.index", fr.frame.payload, { workspaceId: fr.frame.workspaceId });
+    }
+    assert.deepEqual(cp.received.map((f) => [f.type, f.seq]), [["content.index", 1], ["content.index", 2]]);
+    assert.deepEqual(cp.errorsSent, []);
+    env.session.close();
+});
+
+test("invalid-content-index-before-ready: the LA never produces it (no data before ready: not_ready, nothing written)", async () => {
+    const cp = new ReferenceCp();
+    const sent = [];
+    const env = inMemory(cp);
+    env.session.attach({ send: (x) => sent.push(JSON.parse(x)), close() {}, bufferedAmount: () => 0 });
+    const t = traces.find((x) => x.name === "invalid-content-index-before-ready");
+    const bad = t.frames[t.expect.atFrame].frame;
+    await assert.rejects(env.session.enqueue("content.index", bad.payload, { workspaceId: bad.workspaceId }), (e) => e.code === "not_ready");
+    assert.deepEqual(sent.map((f) => f.type), ["register", "hello"]);
+    env.session.close();
+});
+
 test("valid-retransmit-is-idempotent: an ack lost -> the same frame again -> the CP re-acks and counts one event", async () => {
     const cp = new ReferenceCp({ limits: { ackTimeoutMs: 100 } });
     const env = inMemory(cp).start();

package/tools/interop/LaV2NodeContentInteropTest.java

@@ -0,0 +1,262 @@
+package io.visionhammer.gateway.cloud.infrastructure.byoc;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import io.visionhammer.gateway.cloud.application.byoc.v2.LaProtocolV2Properties;
+import io.visionhammer.gateway.cloud.application.byoc.v2.LaV2Chunk;
+import io.visionhammer.gateway.cloud.application.byoc.v2.LaV2InboundListener;
+import io.visionhammer.gateway.cloud.application.byoc.v2.LaV2Metrics;
+import io.visionhammer.gateway.cloud.application.byoc.v2.LaV2Session;
+import io.visionhammer.gateway.cloud.application.byoc.v2.LaV2SessionStore;
+import io.visionhammer.gateway.cloud.application.byoc.v2.LaV2TestKit;
+import io.visionhammer.gateway.cloud.application.content.ContentIndexRegistry;
+import io.visionhammer.gateway.cloud.application.content.ContentRelayProperties;
+import io.visionhammer.gateway.governance.application.DataPlanePolicy;
+import io.visionhammer.gateway.identity.application.AgentAuthorizationEntity;
+import io.visionhammer.gateway.identity.application.AgentAuthorizationStore;
+import io.visionhammer.gateway.shared.localagent.AgentRequestSigner;
+import io.visionhammer.gateway.workspace.application.WorkspaceRepository;
+import org.junit.jupiter.api.Test;
+import org.springframework.http.server.reactive.HttpHandler;
+import org.springframework.http.server.reactive.ReactorHttpHandlerAdapter;
+import org.springframework.web.reactive.socket.server.WebSocketService;
+import org.springframework.web.reactive.socket.server.support.HandshakeWebSocketService;
+import org.springframework.web.reactive.socket.server.upgrade.ReactorNettyRequestUpgradeStrategy;
+import org.springframework.web.server.adapter.WebHttpHandlerBuilder;
+import reactor.core.publisher.Mono;
+import reactor.core.scheduler.Schedulers;
+import reactor.netty.DisposableServer;
+import reactor.netty.http.server.HttpServer;
+
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.time.Clock;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HexFormat;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+
+import static io.visionhammer.gateway.cloud.application.byoc.v2.LaV2TestKit.OM;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assumptions.assumeTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * F3-07 interoperability: the REAL Node Local Agent content handlers (opaque ids, path guard, DLP enforce, git diff,
+ * chunking) against the REAL gateway {@link LaV2Session} behind a real Reactor Netty WebSocket on loopback, real
+ * system clock, real Ed25519 signer. Run from the gateway checkout (JDK 21):
+ *
+ *   LA_REPO=/path/to/deepdebug-local-agent mvn -o test -Dtest=LaV2NodeContentInteropTest
+ *
+ * Skipped when LA_REPO is not set. LA_DLP_MODE=off is the negative control: the agent then registers NO content handler
+ * (F3-07b) and the test fails at the first request. Not proven here: Cloud Run / LB / TLS, Memorystore, two gateway instances, the HTTP
+ * relay endpoint (F3-06) in front of this session.
+ */
+class LaV2NodeContentInteropTest {
+
+    private static final String TENANT = LaV2TestKit.TENANT;
+    private static final String AGENT = LaV2TestKit.AGENT;
+
+    private static Map<String, Object> request(String kind, String ref) {
+        return Map.of("kind", kind, "ref", ref, "actor", Map.of("userId", "usr-42", "role", "member"));
+    }
+
+    @Test
+    void realNodeContentHandlersAgainstTheRealLaV2Session() throws Exception {
+        String laRepo = System.getenv("LA_REPO");
+        assumeTrue(laRepo != null && !laRepo.isBlank(), "LA_REPO not set");
+
+        Clock clock = Clock.systemUTC();
+        LaProtocolV2Properties props = new LaProtocolV2Properties();
+        props.setEnabled(true);
+        List<String> rejected = Collections.synchronizedList(new ArrayList<>());
+        LaV2Metrics metrics = new LaV2Metrics() {
+            @Override public void rejected(String code) { rejected.add(code); }
+        };
+        LaV2InboundListener listener = (t, a, type, rid, payload) -> { };
+        AgentRequestSigner signer = new AgentRequestSigner(LaV2TestKit.PEM, "k1", clock);
+        // F3-07b: the agent announces its ids with content.index; the gateway keeps them in this registry (without a sink it would answer
+        // `rejected protocol_unsupported`, which the metrics below count)
+        ContentRelayProperties relayProps = new ContentRelayProperties();
+        relayProps.setEnabled(true);
+        ContentIndexRegistry index = new ContentIndexRegistry(relayProps, null, System::currentTimeMillis);
+        LaV2Session.Env env = new LaV2Session.Env(LaV2TestKit.CONTRACT, props, signer, OM, clock, metrics, Schedulers.parallel(), List.of(listener), index);
+
+        AgentAuthorizationStore store = mock(AgentAuthorizationStore.class);
+        WorkspaceRepository workspaces = mock(WorkspaceRepository.class);
+        DataPlanePolicy policy = mock(DataPlanePolicy.class);
+        when(workspaces.findDefaultByTenantId(anyString())).thenReturn(Mono.empty());
+        when(store.save(any())).thenAnswer(i -> Mono.just(i.getArgument(0)));
+        when(store.touchLastSeen(any())).thenReturn(Mono.just(1));
+        when(policy.requiresEnforcedSigning(anyString())).thenReturn(Mono.just(true));
+        AgentAuthorizationEntity auth = new AgentAuthorizationEntity();
+        auth.setId(UUID.fromString(AGENT));
+        auth.setTenantId(UUID.fromString(TENANT));
+        auth.setAuthorizationToken("tok");
+        when(store.findByAuthorizationToken("tok")).thenReturn(Mono.just(auth));
+        AgentWebSocketHandler handler = new AgentWebSocketHandler(OM, workspaces, store, signer, null, null, null,
+                new LaV2SessionStore(env), props, policy);
+
+        WebSocketService wsService = new HandshakeWebSocketService(new ReactorNettyRequestUpgradeStrategy());
+        HttpHandler http = WebHttpHandlerBuilder.webHandler(exchange -> wsService.handleRequest(exchange, handler)).build();
+        DisposableServer server = HttpServer.create().host("127.0.0.1").port(0).handle(new ReactorHttpHandlerAdapter(http)).bindNow();
+
+        ProcessBuilder pb = new ProcessBuilder("node", laRepo + "/tools/interop/la-interop-content-client.mjs");
+        pb.environment().putAll(Map.of(
+                "GW_URL", "ws://127.0.0.1:" + server.port() + "/api/v1/agent/ws", "TOKEN", "tok", "TENANT", TENANT, "AGENT", AGENT,
+                "KEY_ID", "k1", "PUBLIC_KEY", LaV2TestKit.PUB, "MODE", "enforce",
+                "LOCAL_AGENT_DLP_MODE", System.getenv().getOrDefault("LA_DLP_MODE", "enforce")));
+        pb.redirectErrorStream(true);
+        Process proc = pb.start();
+        BlockingQueue<JsonNode> events = new LinkedBlockingQueue<>();
+        List<String> transcript = Collections.synchronizedList(new ArrayList<>());
+        Thread reader = new Thread(() -> {
+            try (BufferedReader r = new BufferedReader(new InputStreamReader(proc.getInputStream(), StandardCharsets.UTF_8))) {
+                String line;
+                while ((line = r.readLine()) != null) {
+                    transcript.add(line);
+                    try {
+                        events.add(OM.readTree(line));
+                    } catch (Exception ignored) {
+                        // node warnings
+                    }
+                }
+            } catch (Exception ignored) {
+                // process ended
+            }
+        });
+        reader.setDaemon(true);
+        reader.start();
+        java.io.Writer stdin = new java.io.OutputStreamWriter(proc.getOutputStream(), StandardCharsets.UTF_8);
+
+        try {
+            JsonNode refsEvt = await(events, "refs", 20);
+            await(events, "ready", 15);
+            assertThat(handler.isAgentConnected(TENANT)).isTrue();
+            JsonNode refs = refsEvt.path("refs");
+            String ghp = refsEvt.path("ghp").asText();
+            assertThat(refs.path("envFile").isNull()).as("no id is minted for .env").isTrue();
+            assertThat(refs.path("envBackup").isNull()).as("no id is minted for .env~ (F3-07b)").isTrue();
+            // F3-07b: the ids are the 43-char base64url ones the gateway requires
+            for (String k : List.of("app", "big", "mid", "swapped", "bin", "diffFile")) {
+                assertThat(refs.path(k).asText()).as(k).matches("^[A-Za-z0-9_-]{43}$");
+            }
+
+            // 1. a file with a token: served, the token masked (DLP enforce), the rest intact
+            List<LaV2Chunk> app = content("file", refs.path("app").asText(), handler);
+            assertThat(app.get(0).status()).isEqualTo("ok");
+            String appText = new String(concat(app), StandardCharsets.UTF_8);
+            assertThat(appText).contains("first line").contains("[REDACTED:github_token]").contains("last line").doesNotContain(ghp);
+
+            // 2. 1 MiB = 32 chunks, byte for byte (no secrets in it)
+            List<LaV2Chunk> big = content("file", refs.path("big").asText(), handler);
+            assertThat(big).hasSize(32);
+            assertThat(big.get(31).last()).isTrue();
+            assertThat(HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(concat(big)))).isEqualTo(refsEvt.path("bigSha256").asText());
+            assertThat(concat(big)).hasSize(refsEvt.path("bigBytes").asInt());
+
+            // 3. a token straddling the 32 KiB chunk boundary of the ORIGINAL file never appears, whole or in part
+            List<LaV2Chunk> mid = content("file", refs.path("mid").asText(), handler);
+            assertThat(mid.get(0).status()).isEqualTo("ok");
+            assertThat(mid.size()).isGreaterThanOrEqualTo(2);
+            for (LaV2Chunk c : mid) {
+                assertThat(new String(c.data(), StandardCharsets.UTF_8)).doesNotContain(ghp.substring(0, 12));
+            }
+            assertThat(new String(concat(mid), StandardCharsets.UTF_8)).contains("[REDACTED:github_token]");
+
+            // 4. an id handed out for an innocent file, later swapped for a link to a file outside the workspace: forbidden, no data
+            List<LaV2Chunk> swapped = content("file", refs.path("swapped").asText(), handler);
+            assertThat(swapped).hasSize(1);
+            assertThat(swapped.get(0).status()).isEqualTo("forbidden");
+            assertThat(swapped.get(0).data()).isEmpty();
+
+            // 5. paths are never ids: traversal, absolute, plain relative -> not_found; nothing else comes back
+            for (String ref : List.of("../../etc/passwd", "/etc/passwd", "src/app.txt", "..%2f..%2fetc%2fpasswd", "cr1_AAAAAAAAAAAAAAAAAAAAAA", "A".repeat(43))) {
+                List<LaV2Chunk> r = content("file", ref, handler);
+                assertThat(r).hasSize(1);
+                assertThat(r.get(0).status()).as(ref).isEqualTo("not_found");
+                assertThat(r.get(0).data()).isEmpty();
+            }
+
+            // 6. a binary file cannot be inspected by the DLP: withheld in enforce
+            assertThat(content("file", refs.path("bin").asText(), handler).get(0).status()).isEqualTo("forbidden");
+
+            // 7. kinds the agent does not serve yet: error + protocol_unsupported (a code of the contract)
+            List<LaV2Chunk> report = content("report", refs.path("app").asText(), handler);
+            assertThat(report.get(0).status()).isEqualTo("error");
+            assertThat(report.get(0).errorCode()).isEqualTo("protocol_unsupported");
+
+            // 8. diff of the work tree (symbolic ref, no client data in it) and of one minted file: masked
+            List<LaV2Chunk> diff = content("diff", "worktree", handler);
+            assertThat(diff.get(0).status()).isEqualTo("ok");
+            assertThat(diff.get(0).contentType()).isEqualTo("text/x-diff");
+            String diffText = new String(concat(diff), StandardCharsets.UTF_8);
+            assertThat(diffText).contains("tracked.js").contains("[REDACTED:github_token]").doesNotContain(ghp).doesNotContain("SECRET-ENV-VALUE").doesNotContain(".env");
+            assertThat(new String(concat(content("diff", refs.path("diffFile").asText(), handler)), StandardCharsets.UTF_8)).contains("tracked.js");
+
+            // 9. the consumer cancels after the first chunk (the relay's client went away): the agent is not left blocked
+            List<LaV2Chunk> first = handler.requestContent(TENANT, null, request("file", refs.path("big").asText()), "ws-1")
+                    .take(1).collectList().block(Duration.ofSeconds(20));
+            assertThat(first).hasSize(1);
+            List<LaV2Chunk> afterCancel = content("file", refs.path("app").asText(), handler);
+            assertThat(afterCancel.get(0).status()).isEqualTo("ok");
+
+            // nothing the Node agent put on the wire was refused by the real gateway session
+            assertThat(rejected).as("frames the real LaV2Session rejected: %s", rejected).isEmpty();
+            // nothing in what the agent printed (events, warnings) carries content or a path
+            String printed = String.join("\n", transcript);
+            assertThat(printed).doesNotContain("OUTSIDE-SECRET").doesNotContain("SECRET-ENV-VALUE").doesNotContain("f307-interop-");
+            System.out.println("CONTENT INTEROP OK: rejected=" + rejected);
+        } finally {
+            try {
+                send(stdin, "quit");
+            } catch (Exception ignored) {
+                // gone
+            }
+            proc.waitFor(5, TimeUnit.SECONDS);
+            proc.destroyForcibly();
+            server.disposeNow();
+        }
+    }
+
+    private static List<LaV2Chunk> content(String kind, String ref, AgentWebSocketHandler handler) {
+        return handler.requestContent(TENANT, null, request(kind, ref), "ws-1").collectList().block(Duration.ofSeconds(30));
+    }
+
+    private static void send(java.io.Writer w, String line) throws Exception {
+        w.write(line + "\n");
+        w.flush();
+    }
+
+    private static JsonNode await(BlockingQueue<JsonNode> q, String evt, int seconds) throws Exception {
+        long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(seconds);
+        List<String> seen = new ArrayList<>();
+        while (System.nanoTime() < deadline) {
+            JsonNode n = q.poll(100, TimeUnit.MILLISECONDS);
+            if (n == null) {
+                continue;
+            }
+            if (evt.equals(n.path("evt").asText())) {
+                return n;
+            }
+            seen.add(n.path("evt").asText() + (n.has("code") ? ":" + n.path("code").asText() : "") + (n.has("m") ? ":" + n.path("m").asText() : ""));
+        }
+        throw new AssertionError("no '" + evt + "' event from the Node LA within " + seconds + " s; saw " + seen);
+    }
+
+    private static byte[] concat(List<LaV2Chunk> chunks) {
+        java.io.ByteArrayOutputStream o = new java.io.ByteArrayOutputStream();
+        chunks.forEach(c -> o.writeBytes(c.data()));
+        return o.toByteArray();
+    }
+}