@readymadeit/mcp 0.0.0-stage → 0.1.0

package/bundle/hkp-rt/lib/src/common/stream_broadcast.h

@@ -0,0 +1,250 @@
+#pragma once
+
+#include <algorithm>
+#include <cstddef>
+#include <cstdint>
+#include <deque>
+#include <functional>
+#include <memory>
+#include <mutex>
+#include <vector>
+
+namespace hkp {
+
+// One piece of a stream, shared by every listener it is handed to rather than
+// copied for each.
+using StreamChunk = std::shared_ptr<const std::vector<uint8_t>>;
+
+// Whoever a StreamBroadcast writes to.
+class StreamListener
+{
+public:
+  virtual ~StreamListener() = default;
+
+  // Hands over one chunk. Must return at once: it is called by whoever
+  // produced the chunk, and one listener's connection must never hold up the
+  // stream or the others.
+  virtual void deliver(const StreamChunk& chunk) = 0;
+
+  // Ends this listener's stream. Called once the listener has left, from any
+  // thread.
+  virtual void close() = 0;
+};
+
+// The chunks waiting to be written to one listener.
+//
+// Bounded, because a listener whose connection is slower than the stream would
+// otherwise fall further behind for as long as it stays connected — the gap
+// between what it hears and what is live grows without limit. Past the bound
+// the oldest waiting chunks are dropped: the listener hears a gap and lands
+// back near the live edge. The chunk being written is never dropped, and
+// neither is the newest, so a chunk larger than the bound still goes out.
+class ChunkQueue
+{
+public:
+  explicit ChunkQueue(std::size_t maxBytes) : m_maxBytes(maxBytes) {}
+
+  // Answers how many chunks were dropped to make room.
+  std::size_t push(StreamChunk chunk)
+  {
+    m_bytes += chunk->size();
+    m_chunks.push_back(std::move(chunk));
+
+    std::size_t dropped = 0;
+    const std::size_t firstDroppable = m_writing ? 1 : 0;
+    while (m_bytes > m_maxBytes && m_chunks.size() > firstDroppable + 1)
+    {
+      auto oldest = m_chunks.begin() + firstDroppable;
+      m_bytes -= (*oldest)->size();
+      m_chunks.erase(oldest);
+      ++dropped;
+    }
+    return dropped;
+  }
+
+  // The chunk to write next, now marked as being written — or null when there
+  // is none, or one is already being written.
+  StreamChunk beginWrite()
+  {
+    if (m_writing || m_chunks.empty())
+    {
+      return nullptr;
+    }
+    m_writing = true;
+    return m_chunks.front();
+  }
+
+  // The chunk beginWrite() handed out has been written.
+  void endWrite()
+  {
+    if (!m_writing)
+    {
+      return;
+    }
+    m_writing = false;
+    m_bytes -= m_chunks.front()->size();
+    m_chunks.pop_front();
+  }
+
+  bool writing() const { return m_writing; }
+  std::size_t bytes() const { return m_bytes; }
+  std::size_t size() const { return m_chunks.size(); }
+
+private:
+  std::size_t m_maxBytes;
+  std::size_t m_bytes = 0;
+  bool m_writing = false;
+  std::deque<StreamChunk> m_chunks;
+};
+
+// One stream, written to everyone listening.
+//
+// Every listener gets the same chunks in the same order; there is no stream
+// per listener. A listener that joins late starts with the burst — the most
+// recent chunks, whole, up to `burstBytes` — so a player has something to
+// buffer at once instead of waiting for enough of the stream to arrive. The
+// burst is the only history kept: 0 keeps none, and a newcomer starts at the
+// next chunk.
+//
+// Chunks are the unit throughout — kept, replayed and dropped whole — so a
+// stream whose chunks each start at a point a decoder can start from (whole
+// MP3 frames) stays decodable for everyone.
+//
+// Thread-safe: chunks are published from the pipeline's thread while
+// listeners come and go on the server's.
+class StreamBroadcast
+{
+public:
+  void setBurstBytes(std::size_t bytes)
+  {
+    std::lock_guard<std::mutex> lock(m_mutex);
+    m_burstBytes = bytes;
+    trimBurst();
+  }
+
+  // Called with the listener count whenever it changes, outside the lock.
+  void onListenersChanged(std::function<void(std::size_t)> callback)
+  {
+    std::lock_guard<std::mutex> lock(m_mutex);
+    m_onListenersChanged = std::move(callback);
+  }
+
+  void join(std::shared_ptr<StreamListener> listener)
+  {
+    std::function<void(std::size_t)> changed;
+    std::size_t count = 0;
+    {
+      std::lock_guard<std::mutex> lock(m_mutex);
+      // Under the lock, like publish(), so nothing published meanwhile can
+      // reach this listener ahead of the older chunks in the burst.
+      for (const auto& chunk : m_burst)
+      {
+        listener->deliver(chunk);
+      }
+      m_listeners.push_back(std::move(listener));
+      count = m_listeners.size();
+      changed = m_onListenersChanged;
+    }
+    if (changed)
+    {
+      changed(count);
+    }
+  }
+
+  void leave(const StreamListener* listener)
+  {
+    std::shared_ptr<StreamListener> removed;
+    std::function<void(std::size_t)> changed;
+    std::size_t count = 0;
+    {
+      std::lock_guard<std::mutex> lock(m_mutex);
+      auto it = std::find_if(m_listeners.begin(), m_listeners.end(),
+                             [listener](const auto& entry) { return entry.get() == listener; });
+      if (it == m_listeners.end())
+      {
+        return;
+      }
+      removed = std::move(*it);
+      m_listeners.erase(it);
+      count = m_listeners.size();
+      changed = m_onListenersChanged;
+    }
+    if (changed)
+    {
+      changed(count);
+    }
+  }
+
+  void publish(StreamChunk chunk)
+  {
+    if (!chunk || chunk->empty())
+    {
+      return;
+    }
+    std::lock_guard<std::mutex> lock(m_mutex);
+    if (m_burstBytes > 0)
+    {
+      m_burst.push_back(chunk);
+      m_burstHeld += chunk->size();
+      trimBurst();
+    }
+    for (const auto& listener : m_listeners)
+    {
+      listener->deliver(chunk);
+    }
+  }
+
+  // Ends every listener's stream and forgets the burst.
+  void closeAll()
+  {
+    std::vector<std::shared_ptr<StreamListener>> listeners;
+    std::function<void(std::size_t)> changed;
+    {
+      std::lock_guard<std::mutex> lock(m_mutex);
+      listeners.swap(m_listeners);
+      m_burst.clear();
+      m_burstHeld = 0;
+      changed = m_onListenersChanged;
+    }
+    for (const auto& listener : listeners)
+    {
+      listener->close();
+    }
+    if (changed && !listeners.empty())
+    {
+      changed(0);
+    }
+  }
+
+  std::size_t listenerCount() const
+  {
+    std::lock_guard<std::mutex> lock(m_mutex);
+    return m_listeners.size();
+  }
+
+  // Who is listening right now.
+  std::vector<std::shared_ptr<StreamListener>> listeners() const
+  {
+    std::lock_guard<std::mutex> lock(m_mutex);
+    return m_listeners;
+  }
+
+private:
+  void trimBurst()
+  {
+    while (!m_burst.empty() && m_burstHeld > m_burstBytes)
+    {
+      m_burstHeld -= m_burst.front()->size();
+      m_burst.pop_front();
+    }
+  }
+
+  mutable std::mutex m_mutex;
+  std::vector<std::shared_ptr<StreamListener>> m_listeners;
+  std::deque<StreamChunk> m_burst;
+  std::size_t m_burstHeld = 0;
+  std::size_t m_burstBytes = 0;
+  std::function<void(std::size_t)> m_onListenersChanged;
+};
+
+} // namespace hkp

package/bundle/hkp-rt/lib/src/common/string_util.h

@@ -0,0 +1,31 @@
+#pragma once
+
+#include <string>
+#include <algorithm>
+#include <cstddef>
+
+namespace hkp {
+
+inline std::string toLowerCase(const std::string& method)
+{
+  std::string methodNormalized = method;
+  std::transform(methodNormalized.begin(), methodNormalized.end(), methodNormalized.begin(), ::tolower);
+  return methodNormalized;
+}
+
+// Lowercase hex encoding of a raw byte buffer (e.g. for rendering a digest or
+// random token as text).
+inline std::string toHex(const unsigned char* data, std::size_t len)
+{
+  static const char* const digits = "0123456789abcdef";
+  std::string out;
+  out.reserve(len * 2);
+  for (std::size_t i = 0; i < len; ++i)
+  {
+    out.push_back(digits[data[i] >> 4]);
+    out.push_back(digits[data[i] & 0x0f]);
+  }
+  return out;
+}
+
+}

package/bundle/hkp-rt/lib/src/common/websocket_client_session.cpp

@@ -0,0 +1,217 @@
+#include "./websocket_client_session.h"
+#include "egress.h"
+
+#include <boost/asio.hpp>
+#include <boost/asio/strand.hpp>
+#include <boost/asio/ip/tcp.hpp>
+#include <boost/beast/websocket.hpp>
+
+#include <yas/buffers.hpp>
+
+namespace beast = boost::beast;
+namespace http = beast::http;
+namespace websocket = beast::websocket;
+namespace net = boost::asio;
+using tcp = boost::asio::ip::tcp;
+
+namespace hkp {
+struct WebsocketClientSession::impl
+{
+  impl(boost::asio::io_context &ioc, std::string host, std::string port, std::string path)
+    : m_resolver(net::make_strand(ioc))
+    , m_ws(net::make_strand(ioc))
+    , m_host(host)
+    , m_port(port)
+    , m_path(path)
+    , m_guard(m_ws.get_executor())
+  {
+
+  }
+
+  boost::asio::ip::tcp::resolver m_resolver;
+  beast::websocket::stream<boost::asio::ip::tcp::socket> m_ws;
+  std::string m_host;
+  std::string m_port;
+  std::string m_path;
+  
+  std::atomic_flag m_isSending = ATOMIC_FLAG_INIT;
+  std::atomic_flag m_isConnected = ATOMIC_FLAG_INIT;
+  boost::asio::executor_work_guard<beast::websocket::stream<boost::asio::ip::tcp::socket>::executor_type> m_guard;
+
+  beast::flat_buffer m_incomingBuffer;
+  CallbackFunc m_callback;
+};
+
+
+WebsocketClientSession::WebsocketClientSession(boost::asio::io_context &ioc, std::string host, std::string port, std::string path)
+: m_impl(std::make_unique<impl>(ioc, host, port, path))
+{
+}
+
+WebsocketClientSession::~WebsocketClientSession()
+{
+
+}
+
+bool WebsocketClientSession::isConnected() const
+{
+  return m_impl->m_isConnected.test();
+}
+
+void WebsocketClientSession::signalSend(const Data& data)
+{
+  if (!isConnected())
+  {
+    std::cout << "WebsocketClientSession::signalSend: Send binary skipped because not connected" << std::endl;
+    return;
+  }
+
+  if (m_impl->m_isSending.test())
+  {
+    std::cout << "\rWebsocketClientSession::signalSend: Send binary skipped because already sending  ..." << std::endl;
+    return;
+  }
+
+  boost::asio::post(m_impl->m_ws.get_executor(),
+                    [this, data]()
+                    { send_data(data); });
+}
+
+void WebsocketClientSession::send_data(const Data& data)
+{
+  if (auto buffer = getRingBufferFromData(data)) 
+  {
+    auto numAvailable = buffer->availableCount();
+    if (numAvailable > 0)
+    {
+      m_impl->m_ws.binary(true);
+      buffer->resyncIfNeeded();
+      auto buf = buffer->serialise(); 
+      m_impl->m_isSending.test_and_set();
+      m_impl->m_ws.write(
+          boost::asio::buffer(buf.data.get(), buf.size));
+      m_impl->m_isSending.clear();
+    }
+    else
+    {
+      std::cout << "WebsocketClientSession::signalSend: Not enough data in RingBuffer available" << std::endl;
+    }
+  }
+  else if (auto binary = getBinaryFromData(data))
+  {
+    m_impl->m_ws.binary(true);
+    m_impl->m_isSending.test_and_set();
+    m_impl->m_ws.write(net::buffer(&(*binary)[0], binary->size()));
+    m_impl->m_isSending.clear();
+  }
+  else if (isUndefined(data) || isNull(data))
+  {
+    std::cout << "WebsocketClientSession::signalSend: Undefined or null data currently not available" << std::endl;
+  }
+  else if (auto str = getStringFromData(data))
+  {
+    m_impl->m_ws.text(true);
+    m_impl->m_isSending.test_and_set();
+    m_impl->m_ws.write(net::buffer(*str));
+    m_impl->m_isSending.clear();
+  }
+  else if (auto json = getJSONFromData(data))
+  {
+    m_impl->m_ws.text(true);
+    m_impl->m_isSending.test_and_set();
+    m_impl->m_ws.write(net::buffer(json->dump()));
+    m_impl->m_isSending.clear();
+  }
+  else 
+  {
+    std::cout << "WebsocketClientSession::signalSend: Data type not supported available: " << data.which() << std::endl;
+  }
+}
+
+void WebsocketClientSession::start(std::string id, CallbackFunc callback)
+{
+  m_impl->m_callback = callback;
+
+  auto const results = ::hkp::egress::resolve(m_impl->m_resolver, m_impl->m_host, m_impl->m_port);
+
+  // Make the connection on the IP address we get from a lookup
+  auto ep = net::connect(m_impl->m_ws.next_layer(), results);
+  auto hostAndPort = m_impl->m_host + ':' + std::to_string(ep.port());
+
+  // Set a decorator to change the User-Agent of the handshake
+  m_impl->m_ws.set_option(websocket::stream_base::decorator(
+      [](websocket::request_type &req)
+      {
+        req.set(http::field::user_agent,
+                std::string(BOOST_BEAST_VERSION_STRING) +
+                    " websocket-client-coro");
+      }));
+
+  m_impl->m_ws.handshake(hostAndPort, m_impl->m_path);
+  m_impl->m_isConnected.test_and_set();
+
+  std::string protocol = json{{"id", id}, {"type", "readWrite"}}.dump();
+  m_impl->m_ws.write(net::buffer(protocol));
+
+  m_impl->m_ws.async_read(
+    m_impl->m_incomingBuffer,
+      beast::bind_front_handler(
+            &WebsocketClientSession::on_read,
+          shared_from_this()));
+}
+
+void WebsocketClientSession::stop()
+{
+  if (m_impl->m_isConnected.test_and_set())
+  {
+    m_impl->m_guard.reset();
+    m_impl->m_ws.close(websocket::close_code::normal);
+  }
+}
+
+void WebsocketClientSession::close()
+{
+  // Close the WebSocket connection
+  m_impl->m_ws.async_close(websocket::close_code::normal,
+                    beast::bind_front_handler(
+                        &WebsocketClientSession::on_close,
+                        shared_from_this()));
+}
+
+void WebsocketClientSession::on_close(beast::error_code ec)
+{
+  if (ec)
+  {
+    return fail(ec, "close fail");
+  }
+  std::cout << "Writer session - closed connection" << std::endl;
+}
+
+void WebsocketClientSession::on_read(beast::error_code ec, std::size_t bytes_transferred)
+{
+  if (ec)
+  {
+    return fail(ec, "WebsocketClientSession::on_read");
+  }
+
+  if (m_impl->m_callback)
+  {
+    auto isBinary = m_impl->m_ws.got_binary();
+    m_impl->m_callback(m_impl->m_incomingBuffer, bytes_transferred, isBinary);
+  }
+
+  m_impl->m_incomingBuffer.consume(bytes_transferred);
+
+  m_impl->m_ws.async_read(
+    m_impl->m_incomingBuffer,
+      beast::bind_front_handler(
+            &WebsocketClientSession::on_read,
+          shared_from_this()));
+}
+
+void WebsocketClientSession::fail(beast::error_code ec, char const *what)
+{
+  std::cerr << what << ": " << ec.message() << "\n";
+}
+
+}

package/bundle/hkp-rt/lib/src/common/websocket_client_session.h

@@ -0,0 +1,37 @@
+#pragma once
+
+#include <boost/beast.hpp>
+
+#include <types/types.h>
+
+namespace hkp {
+  
+class WebsocketClientSession : public std::enable_shared_from_this<WebsocketClientSession>
+{
+public:
+  typedef std::function<void(boost::beast::flat_buffer& buffer, std::size_t bytes_transferred, bool isBinary)> CallbackFunc;
+
+  WebsocketClientSession(boost::asio::io_context &ioc, std::string host, std::string port, std::string path);
+  ~WebsocketClientSession();
+
+  bool isConnected() const;
+  void signalSend(const Data& data);
+
+  void start(std::string id, CallbackFunc callback);
+  void stop();
+
+  void close();
+
+private:
+  void fail(boost::beast::error_code ec, char const *what);
+  void on_close(boost::beast::error_code ec);
+  void send_data(const Data& data);
+
+  void on_read(boost::beast::error_code ec, std::size_t bytes_transferred);
+
+private:
+  struct impl;
+  std::unique_ptr<impl> m_impl;
+};
+
+}

package/bundle/hkp-rt/lib/src/common/websocket_config.h

@@ -0,0 +1,119 @@
+#pragma once
+
+#include <string>
+#include <types/types.h>
+
+namespace hkp {
+struct WebsocketConfig
+{
+  std::string host;
+  std::string port;
+  std::string path;
+
+  static WebsocketConfig withHost(const std::string& host)
+  {
+    WebsocketConfig config;
+    config.host = host;
+    return config;
+  }
+
+  static std::optional<WebsocketConfig> fromJSON(const json& buf)
+  {
+    WebsocketConfig config;
+    if (!config.load(buf)) 
+    {
+      return std::nullopt;
+    }
+
+    return std::optional<WebsocketConfig>(config);
+  }
+
+  WebsocketConfig& withPath(const std::string& p)
+  {
+    this->path = p;
+    return *this;
+  }
+
+  WebsocketConfig& withPort(const std::string& p)
+  {
+    this->port = p;
+    return *this;
+  }
+  
+  WebsocketConfig& withRandomPort()
+  {
+    this->port = "0";
+    return *this;
+  }
+
+  bool update(json& buf)
+  {
+    if (buf.is_null())
+    {
+      return false;
+    }
+    if (buf.contains("host"))
+    {
+      host = buf["host"];
+    }
+    if (buf.contains("port"))
+    {
+      port = buf["port"];
+    }
+    if (buf.contains("path"))
+    {
+      path = buf["path"];
+    }
+
+    return isValid();
+  }
+
+  unsigned short portAsNumber() const
+  {
+    if (port.empty())
+    {
+      return 0;
+    }
+    return std::stoi(port);
+  }
+
+  bool isValid() const
+  {
+    return !host.empty() && !port.empty() && !path.empty();
+  }
+
+  bool load(json buf)
+  {
+    if (buf.is_null())
+    {
+      return false;
+    }   
+    bool updated = updateIfNeeded(host, buf["host"]);
+    updated |= updateIfNeeded(port, buf["port"]);
+    updated |= updateIfNeeded(path, buf["path"]);
+    return updated;
+  }
+
+  json serialise() const
+  {
+    json buf;
+    buf["host"] = host;
+    buf["port"] = port;
+    buf["path"] = path;
+    return buf;
+  }
+
+  std::string toString() const
+  {
+    return host + ":" + port + path;
+  }
+};
+
+
+inline std::ostream& operator<<(std::ostream& os, const WebsocketConfig& c)
+{
+  os << "Host: " << c.host << " Path: " << c.path << " Port: " << c.port << std::endl;
+  return os;
+} 
+
+}

package/bundle/hkp-rt/lib/src/common/websocket_protocol.h

@@ -0,0 +1,32 @@
+#pragma once
+
+#include <string>
+
+#include <nlohmann/json.hpp>
+
+namespace hkp {
+
+// The protocol specifies the communiation type of the client,
+// e.g. "writer", "reader", or "readwrite". 
+// The id is used to identify the client.
+
+struct WebsocketProtocol
+{
+  static WebsocketProtocol parse(const std::string& s)
+  {
+    auto buf = nlohmann::json::parse(s);
+    WebsocketProtocol protocol;
+    if (!buf.contains("id") || !buf.contains("type"))
+    {
+      throw std::runtime_error("WebsocketProtocol::parse: invalid protocol");
+    }
+    protocol.id = buf["id"];
+    protocol.type = buf["type"];
+    return protocol;
+  }
+
+  std::string id;
+  std::string type;
+};
+
+}