@volter/twin-livekit 0.1.0

package/test-fixtures/protobufs/livekit_agent_dispatch.proto

@@ -0,0 +1,103 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto3";
+
+package livekit;
+option go_package = "github.com/livekit/protocol/livekit";
+option csharp_namespace = "LiveKit.Proto";
+option ruby_package = "LiveKit::Proto";
+
+import "livekit_agent.proto";
+import "logger/options.proto";
+
+enum JobRestartPolicy {
+  JRP_ON_FAILURE = 0; // restart when the job fails (default)
+  JRP_NEVER = 1;      // never restart
+}
+
+service AgentDispatchService {
+  rpc CreateDispatch(CreateAgentDispatchRequest) returns (AgentDispatch);
+  rpc DeleteDispatch(DeleteAgentDispatchRequest) returns (AgentDispatch);
+  rpc ListDispatch(ListAgentDispatchRequest) returns (ListAgentDispatchResponse);
+}
+
+message CreateAgentDispatchRequest {
+  string agent_name = 1;
+  string room = 2;
+  string metadata = 3 [
+    (logger.sensitivity) = SENSITIVITY_PII,
+    (logger.redact_format) = "<redacted ({{ .Size }} bytes)>"
+  ];
+  JobRestartPolicy restart_policy = 4; // cloud only
+  string deployment = 5;
+  map<string, string> attributes = 6 [
+    (logger.sensitivity) = SENSITIVITY_PII,
+    (logger.redact_format) = "<redacted ({{ .Size }} bytes)>"
+  ];
+}
+
+message RoomAgentDispatch {
+  string agent_name = 1;
+  string metadata = 2 [
+    (logger.sensitivity) = SENSITIVITY_PII,
+    (logger.redact_format) = "<redacted ({{ .Size }} bytes)>"
+  ];
+  JobRestartPolicy restart_policy = 3; // cloud only
+  string deployment = 4;
+  map<string, string> attributes = 5 [
+    (logger.sensitivity) = SENSITIVITY_PII,
+    (logger.redact_format) = "<redacted ({{ .Size }} bytes)>"
+  ];
+}
+
+message DeleteAgentDispatchRequest {
+  string dispatch_id = 1 [(logger.name) = "dispatchID"];
+  string room = 2;
+}
+
+message ListAgentDispatchRequest {
+  string dispatch_id = 1 [(logger.name) = "dispatchID"]; // if set, only the dispatch whose id is given will be returned
+  string room = 2;        // name of the room to list agents for. Must be set.
+}
+
+message ListAgentDispatchResponse {
+  repeated AgentDispatch agent_dispatches = 1;
+}
+
+message AgentDispatch {
+  string id = 1;
+  string agent_name = 2;
+  string room = 3;
+  string metadata = 4 [
+    (logger.sensitivity) = SENSITIVITY_PII,
+    (logger.redact_format) = "<redacted ({{ .Size }} bytes)>"
+  ];
+  AgentDispatchState state = 5;
+  JobRestartPolicy restart_policy = 6; // cloud only
+  string deployment = 7;
+  map<string, string> attributes = 8 [
+    (logger.sensitivity) = SENSITIVITY_PII,
+    (logger.redact_format) = "<redacted ({{ .Size }} bytes)>"
+  ];
+}
+
+message AgentDispatchState {
+  // For dispatches of tyoe JT_ROOM, there will be at most 1 job.
+  // For dispatches of type JT_PUBLISHER, there will be 1 per publisher.
+  repeated Job jobs = 1;
+  int64 created_at = 2;
+  int64 deleted_at = 3;
+}
+

package/test-fixtures/protobufs/livekit_agent_simulation.proto

@@ -0,0 +1,422 @@
+// Copyright 2024 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto3";
+
+package livekit;
+
+import "agent/livekit_agent_session.proto";
+import "google/protobuf/duration.proto";
+import "google/protobuf/timestamp.proto";
+import "livekit_cloud_agent.proto";
+import "livekit_models.proto";
+
+option csharp_namespace = "LiveKit.Proto";
+option go_package = "github.com/livekit/protocol/livekit";
+option ruby_package = "LiveKit::Proto";
+
+message SimulationRunSummary {
+  int32 passed = 1;
+  int32 failed = 2;
+  string going_well = 3;
+  string to_improve = 4;
+  repeated Issue issues = 5;
+  map<string, livekit.agent.ChatContext> chat_history = 6;
+
+  message Issue {
+    string description = 1;
+    string suggestion = 2;
+    string label = 3;
+  }
+}
+
+message SimulationRun {
+  enum Status {
+    STATUS_PENDING_UPLOAD = 0;
+    STATUS_GENERATING = 1;
+    STATUS_RUNNING = 2;
+    STATUS_SUMMARIZING = 3;
+    STATUS_COMPLETED = 4;
+    STATUS_FAILED = 5;
+    STATUS_CANCELLED = 6;
+  }
+
+  message Job {
+    enum Status {
+      STATUS_PENDING = 0;
+      STATUS_RUNNING = 1;
+      STATUS_COMPLETED = 2;
+      STATUS_FAILED = 3;
+      STATUS_CANCELLED = 4;
+    }
+    string id = 1;
+    Status status = 2;
+    string instructions = 3;
+    string error = 4;
+    string agent_expectations = 5;
+    string label = 6;
+    repeated string tags = 7;
+    string room_name = 8;
+    reserved 9;  // was scenario_id (scenarios are no longer stored)
+    google.protobuf.Timestamp started_at = 10;
+    google.protobuf.Timestamp ended_at = 11;
+    string room_id = 12;
+    Usage usage = 13;
+    // Quality metrics for this job's call. Unset when the job produced none.
+    JobMetrics metrics = 14;
+
+    message Usage {
+      int32 text_turns_count = 1;
+      int32 audio_turns_count = 2;
+    }
+  }
+
+  message JobMetrics {
+    // Headline scores, 0-1. All absent when the simulator spoiled the call.
+    optional float accuracy_score = 1;
+    optional float experience_score = 2;
+    optional float task_completion = 3;  // the scenario verdict as 1/0
+
+    STT stt = 4;
+    LLM llm = 5;
+    TTS tts = 6;
+    Conversation conversation = 7;
+
+    // One entry per turn, both speakers.
+    repeated Turn turns = 9;
+
+    string judge_model = 10;            // text judge for judged scores; "" if none ran
+    bool has_remote_session = 12;       // false = waveform-only capture (e.g. SIP)
+    google.protobuf.Timestamp t0 = 13;  // audio only; call start (first speech edge); turn times relative to it
+
+    // Judged over the authored dialog (text and audio); needs the text judge.
+    optional float conciseness = 14;  // judged {0, 0.5, 1} per agent turn; job = mean
+    // Whole-call issue flags; true = present, absent = not judged.
+    optional bool unnecessary_tool_calls = 15;
+    optional bool information_loss = 16;
+    optional bool redundant_statements = 17;
+    optional bool poor_question_quality = 18;
+    optional float conversation_progression = 19;  // 0-1, derived from the flags above
+
+    // The agent's perception of the caller. Audio only.
+    message STT {
+      optional float wer = 1;                       // needs the RemoteSession; word_errors / words
+      optional uint32 words = 2;                    // needs the RemoteSession
+      optional uint32 word_errors = 3;              // needs the RemoteSession
+      optional float cer = 4;                       // needs the RemoteSession; char_errors / chars
+      optional uint32 chars = 9;                    // needs the RemoteSession
+      optional uint32 char_errors = 10;             // needs the RemoteSession
+      optional uint32 stt_delay_ms = 8;             // needs the RemoteSession; agent-reported STT/endpointing delay
+
+      // Key entities (names, IDs, amounts) the caller said. Needs the text judge.
+      optional float entity_recognition = 11;  // recognized / uttered
+      optional uint32 entities_uttered = 12;
+      optional uint32 entities_recognized = 13;
+      // Retention: an entity is acquired at the first saying heard right, and
+      // these count only the sayings after that, so recall separates keeping an
+      // entity from ever getting it.
+      optional float entity_recall = 14;  // acquired_recognized / acquired_uttered
+      optional uint32 entities_acquired_uttered = 15;
+      optional uint32 entities_acquired_recognized = 16;
+    }
+
+    message LLM {
+      optional uint32 ttft_ms = 1;            // needs the RemoteSession; time to first token
+      optional uint32 ttfs_ms = 2;            // needs the RemoteSession; time to first sentence, the smallest speakable unit
+      optional float tokens_per_second = 3;   // needs the RemoteSession
+    }
+
+    // The agent's voice. Audio only.
+    message TTS {
+      optional uint32 ttfb_ms = 2;             // needs the RemoteSession; provider byte-level TTFB
+      optional float wer = 3;                  // needs the RemoteSession; word_errors / words
+      optional uint32 words = 7;               // needs the RemoteSession
+      optional uint32 word_errors = 8;         // needs the RemoteSession
+      optional float cer = 9;                  // needs the RemoteSession; char_errors / chars
+      optional uint32 chars = 10;              // needs the RemoteSession
+      optional uint32 char_errors = 11;        // needs the RemoteSession
+
+      // Key entities the agent said, as the caller heard them. Needs the text judge.
+      optional float entity_recognition = 14;  // recognized / uttered
+      optional uint32 entities_uttered = 15;
+      optional uint32 entities_recognized = 16;
+      optional float entity_recall = 17;  // acquired_recognized / acquired_uttered
+      optional uint32 entities_acquired_uttered = 18;
+      optional uint32 entities_acquired_recognized = 19;
+    }
+
+    // Audio only.
+    message Conversation {
+      optional float turn_taking_score = 1;              // 0-1
+      optional int32 heard_e2e_latency_p50_ms = 2;       // floor-transfer offset; a gap is > 0, an overlap < 0
+      optional int32 heard_e2e_latency_p95_ms = 3;
+      optional int32 heard_e2e_latency_p99_ms = 4;
+      optional uint32 time_to_yield_ms = 5;              // agent audio continuing past a barge-in
+      optional uint32 eot_misprediction_count = 6;       // agent started before the caller's turn ended
+      optional float overlap_ratio = 7;                  // overlapping speech / total speech
+      optional uint32 overlap_speech_ms = 8;             // pooling stats for overlap_ratio
+      optional uint32 total_speech_ms = 9;
+      optional uint32 awkward_silence_count = 11;        // gaps past the natural-pause threshold
+      optional uint32 unanswered_turns = 12;             // the simulated party spoke, the agent never responded
+      optional uint32 false_interruption_count = 13;     // needs the RemoteSession; agent paused for a non-interruption
+      optional uint32 e2e_latency_ms = 15;               // needs the RemoteSession; the agent's own claim
+    }
+
+    // Rows exist for text and audio. The groups above hold the means and pools
+    // over these; anything not measured for a turn stays unset.
+    message Turn {
+      uint32 index = 1;                 // 1-based, conversation order
+      livekit.agent.ChatRole role = 2;  // ASSISTANT = the agent under test, USER = simulator persona
+      optional uint32 start_ms = 3;     // audio only; relative to t0; unset if the turn could not be aligned to the audio
+      optional uint32 end_ms = 4;       // audio only
+
+      optional uint32 stt_delay_ms = 5;    // needs the RemoteSession
+      optional uint32 llm_ttft_ms = 6;     // needs the RemoteSession
+      optional uint32 llm_ttfs_ms = 7;     // needs the RemoteSession
+      optional float llm_tps = 8;          // needs the RemoteSession
+      optional uint32 tts_ttfb_ms = 9;     // needs the RemoteSession
+      optional uint32 e2e_latency_ms = 10; // needs the RemoteSession
+
+      optional int32 heard_e2e_latency_ms = 11;
+      optional uint32 time_to_yield_ms = 12;
+      optional float turn_taking_score = 13;
+      optional float conciseness = 14;     // needs the text judge
+
+      optional bool eot_misprediction = 15;
+      optional bool awkward_silence = 16;
+      optional bool unanswered = 17;
+    }
+  }
+
+  // Aggregates over this run's jobs, so a run summary does not have to pull
+  // every job and turn. Reuses the JobMetrics group shapes.
+  message RunMetrics {
+    optional float accuracy_score = 1;     // mean over scored jobs
+    optional float experience_score = 2;
+    optional float scenario_pass_rate = 3; // share of jobs whose scenario verdict passed
+
+    JobMetrics.STT stt = 4;
+    JobMetrics.LLM llm = 5;
+    JobMetrics.TTS tts = 6;
+    JobMetrics.Conversation conversation = 7;
+
+    // Means over the jobs that report them.
+    optional float conciseness = 8;
+    optional float conversation_progression = 9;
+  }
+
+  message Create {
+    message Request {
+      reserved 3;  // was agent_description (input never consumed; description is derived from source)
+      reserved "agent_description";
+      string project_id = 1;
+      string agent_name = 2;
+      int32 num_simulations = 4;
+      string region = 6;
+
+      // When set, run these scenarios (loaded from a scenarios.yaml).
+      // When unset, generate num_simulations scenarios from the uploaded agent source.
+      optional ScenarioGroup scenario_group = 7;
+
+      // Maximum simulate jobs running in parallel for this run. Clamped to the
+      // project's concurrency quota; 0/unset uses the server default.
+      optional int32 concurrency = 8;
+
+      // Conversation mode for every job in this run; unspecified = TEXT.
+      SimulationMode mode = 9;
+
+      // Impairments applied to the simulated user's audio; AUDIO mode only.
+      bool background_noise = 10;
+      bool low_quality_microphone = 11;
+      bool packet_loss = 12;
+    }
+    message Response {
+      string simulation_run_id = 1;
+      PresignedPostRequest presigned_post_request = 2;
+    }
+  }
+
+  message ConfirmSourceUpload {
+    message Request {
+      string project_id = 1;
+      string simulation_run_id = 2;
+      string code_entrypoint = 3;
+    }
+    message Response {}
+  }
+
+  message Get {
+    message Request {
+      string project_id = 1;
+      string simulation_run_id = 2;
+    }
+    message Response {
+      SimulationRun run = 1;
+    }
+  }
+
+  message List {
+    message Request {
+      string project_id = 1;
+      optional Status status = 2;
+      optional TokenPagination page_token = 3;
+      // Half-open window [start_time, end_time) on created_at, the leading key
+      // of this list's keyset. Unset bounds mean no window, so existing callers
+      // keep their unbounded scan; the histogram's selected range sets both.
+      optional google.protobuf.Timestamp start_time = 4;
+      optional google.protobuf.Timestamp end_time = 5;
+      // Exact agent name. Unset means every agent — including the runs whose
+      // name is empty, which a bare string could not tell from no filter.
+      optional string agent_name = 6;
+      // Absent applies no mode filter. TEXT matches runs stored as TEXT or
+      // UNSPECIFIED, which `SimulationRun.mode` defines as the same mode.
+      optional SimulationMode mode = 7;
+    }
+    message Response {
+      repeated SimulationRun runs = 1;
+      TokenPagination next_page_token = 2;
+    }
+  }
+
+  // Counts of the runs `List` would return over the same window, bucketed by
+  // created_at instead of paginated.
+  message Counts {
+    message Request {
+      string project_id = 1;
+      // Both bounds required: a bucketed scan of a project's whole history is
+      // never what a caller wants.
+      google.protobuf.Timestamp start_time = 2;
+      google.protobuf.Timestamp end_time = 3;
+      // Applied as `List` applies it: a filter the rows honour and the bars
+      // above them drop would make the histogram an answer about a wider set.
+      optional Status status = 4;
+      optional string agent_name = 5;
+      optional SimulationMode mode = 6;
+    }
+    message Response {
+      // Buckets holding at least one run, ascending by bucket_start. Empty
+      // buckets are omitted; each run lands in exactly one bucket, so the
+      // counts sum to the number of runs in the window.
+      repeated Bucket buckets = 1;
+      // The bucket width the server chose from the window, so the client
+      // zero-fills the omitted buckets on the grid the counts were built on.
+      google.protobuf.Duration interval = 2;
+    }
+    message Bucket {
+      google.protobuf.Timestamp bucket_start = 1;
+      uint64 count = 2;
+    }
+  }
+
+  message Cancel {
+    message Request {
+      string project_id = 1;
+      string simulation_run_id = 2;
+    }
+    message Response {}
+  }
+
+  string id = 1;
+  string project_id = 2;
+  Status status = 3;
+  string agent_description = 4;
+  string error = 5;
+  google.protobuf.Timestamp created_at = 6;
+  repeated Job jobs = 7;
+  // was: SimulationRunSummary summary (replaced by summary_zstd)
+  reserved 8;
+  reserved "summary";
+  string agent_name = 9;
+  ScenarioGroup scenario_group = 10;
+  google.protobuf.Timestamp ended_at = 11;
+  int32 job_count = 12;
+  int32 passed_count = 13;
+  int32 failed_count = 14;
+  int32 num_simulations = 15;
+  // Aggregate usage across all jobs in this run (sum of Job.usage).
+  Usage usage = 16;
+  // Maximum simulate jobs running in parallel for this run (0 = server default).
+  int32 concurrency = 17;
+  // Conversation mode for every job in this run; unspecified = TEXT.
+  SimulationMode mode = 18;
+  // Run-level aggregates over this run's jobs.
+  RunMetrics metrics = 19;
+  // zstd-compressed SimulationRunSummary: decompress, then proto.Unmarshal.
+  // Compressed by default so the blob ships as-is on every hop.
+  bytes summary_zstd = 20;
+
+  message Usage {
+    int32 text_turns_count = 1;
+    int32 audio_turns_count = 2;
+  }
+}
+
+// A single scenario, mirroring one entry in a scenarios.yaml file. Scenarios
+// are no longer stored server-side; the yaml file is the source of truth.
+message Scenario {
+  message CreateFromSession {
+    message Request {
+      string project_id = 1;
+      string room_id = 2;
+      string region = 3;
+    }
+    message Response {
+      Scenario scenario = 1;
+    }
+  }
+
+  string label = 1;
+  string instructions = 2;
+  string agent_expectations = 3;
+  // Set as participant attributes on the simulation participant's access token.
+  map<string, string> tags = 4;
+  // Arbitrary JSON-encoded object, surfaced to the agent under test as
+  // SimulationContext userdata (e.g. inputs and benchmark target state).
+  string userdata = 5;
+}
+
+// A named group of scenarios, mirroring a whole scenarios.yaml file.
+message ScenarioGroup {
+  string name = 1;
+  repeated Scenario scenarios = 2;
+}
+
+// How the simulated user interacts with the agent under test.
+enum SimulationMode {
+  SIMULATION_MODE_UNSPECIFIED = 0;
+  // The simulator chats over text streams (no audio).
+  SIMULATION_MODE_TEXT = 1;
+  // The simulator publishes/subscribes audio in the room.
+  SIMULATION_MODE_AUDIO = 2;
+}
+
+// Carried (protojson-encoded) in the simulation room's metadata so the agent
+// under test can read the scenario + run identity synchronously at connect.
+message SimulationDispatch {
+  string simulation_run_id = 1;
+  string job_id = 2;
+  Scenario scenario = 3;
+  SimulationMode mode = 4;
+}
+
+service AgentSimulation {
+  rpc CreateSimulationRun(SimulationRun.Create.Request) returns (SimulationRun.Create.Response);
+  rpc ConfirmSimulationSourceUpload(SimulationRun.ConfirmSourceUpload.Request) returns (SimulationRun.ConfirmSourceUpload.Response);
+  rpc GetSimulationRun(SimulationRun.Get.Request) returns (SimulationRun.Get.Response);
+  rpc ListSimulationRuns(SimulationRun.List.Request) returns (SimulationRun.List.Response);
+  rpc CountSimulationRuns(SimulationRun.Counts.Request) returns (SimulationRun.Counts.Response);
+  rpc CancelSimulationRun(SimulationRun.Cancel.Request) returns (SimulationRun.Cancel.Response);
+
+  rpc CreateScenarioFromSession(Scenario.CreateFromSession.Request) returns (Scenario.CreateFromSession.Response);
+}

package/test-fixtures/protobufs/livekit_agent_worker.proto

@@ -0,0 +1,29 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto3";
+
+package livekit;
+option go_package = "github.com/livekit/protocol/livekit";
+option csharp_namespace = "LiveKit.Proto";
+option ruby_package = "LiveKit::Proto";
+
+message WorkerInfo {
+  string worker_type     = 1;
+  string agent_name      = 2;
+  float active_jobs      = 3;
+  string sdk_version     = 4;
+  float worker_load      = 5;
+  int32 protocol_version = 6;
+}