@readymadeit/mcp 0.0.0-stage → 0.1.0

package/bundle/hkp-python/src/hkp/ws_tickets.py

@@ -0,0 +1,54 @@
+"""Single-use tickets that open a WebSocket in place of a token.
+
+A browser cannot set a header on a WebSocket handshake, so a token it
+authenticated one with would have to ride in the address — and an address is
+what proxies and access logs keep. So a browser first asks for a ticket
+(``POST /ws-tickets``, with its token in the ``Authorization`` header like any
+request) and opens the socket with ``?ticket=`` instead. A ticket opens one
+socket, within seconds, as whoever asked for it, and is worth nothing once
+written down.
+
+The same in hkp-node (``src/wsTickets.ts``) and hkp-rt (``http/server.cpp``).
+"""
+
+from __future__ import annotations
+
+import secrets
+import time
+from collections import OrderedDict
+from typing import Callable, Generic, TypeVar
+
+# Long enough for a browser to open the socket it asked for, and no longer.
+TICKET_TTL_SECONDS = 30.0
+# Outstanding tickets kept; past this the oldest go first.
+_MAX_OUTSTANDING = 10_000
+
+Holder = TypeVar("Holder")
+
+
+class WsTickets(Generic[Holder]):
+    def __init__(self, now: Callable[[], float] = time.monotonic) -> None:
+        self._now = now
+        self._tickets: OrderedDict[str, tuple[Holder, float]] = OrderedDict()
+
+    def mint(self, holder: Holder) -> str:
+        """A ticket for ``holder``, good for one socket within the TTL."""
+        self._prune()
+        while len(self._tickets) >= _MAX_OUTSTANDING:
+            self._tickets.popitem(last=False)
+        ticket = secrets.token_urlsafe(32)
+        self._tickets[ticket] = (holder, self._now() + TICKET_TTL_SECONDS)
+        return ticket
+
+    def redeem(self, ticket: str) -> Holder | None:
+        """Who a ticket was minted for — once. Unknown, used or expired: None."""
+        held = self._tickets.pop(ticket, None)
+        if held is None:
+            return None
+        holder, expires = held
+        return holder if expires >= self._now() else None
+
+    def _prune(self) -> None:
+        now = self._now()
+        for ticket in [t for t, (_, expires) in self._tickets.items() if expires < now]:
+            del self._tickets[ticket]

package/bundle/hkp-python/src/hkp/yas.py

@@ -0,0 +1,192 @@
+from __future__ import annotations
+
+"""YAS binary wire format — Python port.
+
+Reference implementations:
+  hkp-frontend/src/runtime/rest/Message.ts   (TypeScript, hand-rolled)
+  hkp-rt/lib/src/types/message.cpp           (C++, yas library)
+
+Frame layout (all integers little-endian):
+
+  7 bytes   YAS header  b"yas" + 4 flag/version bytes (validated on 'yas' only)
+  uint16    message purpose
+  uint16    data type id (DataTypeId)
+  uint64    sender length
+  bytes     sender (ascii)
+  ...       payload, type-dependent:
+
+  FloatRingBuffer:  nested 7-byte YAS header
+                    + type id            <- DIALECT: uint16 (C++ writer) or
+                                            uint32 (TS writer); auto-detected
+                                            on read via total-length check
+                    + uint64 numSamples
+                    + numSamples * 4 bytes of float32 samples
+                    + uint32 id
+                    + uint64 ts
+  JSON / String:    nested 7-byte YAS header + uint64 byteLength + utf-8 bytes
+  BinaryData:       raw bytes to end of frame (no nested header)
+  Null/Undefined:   single ignored byte
+
+On write this codec emits the uint16 ring-buffer dialect, which is what the
+TypeScript deserializer expects — every runtime hop is mediated by the
+frontend, so the frontend is our only YAS peer in practice.
+"""
+
+import json
+import struct
+from dataclasses import dataclass
+from typing import Any
+
+from .data import (
+    BinaryData,
+    DataTypeId,
+    FloatRingBuffer,
+    NullData,
+    TextData,
+    UndefinedData,
+    get_data_type_id,
+)
+
+
+class MessagePurpose:
+    NOTIFICATION = 0
+    RESULT = 1
+    RESULT_AWAITING_RESPONSE = 2
+    RESULT_WITH_REQUEST_ID = 3
+
+
+# Matches the header bytes the TypeScript writer emits ("yas" 0 0 1 7); the
+# C++ yas library accepts these, and readers only validate the "yas" magic.
+YAS_HEADER = b"yas0017"
+
+
+@dataclass
+class Message:
+    purpose: int
+    sender: str
+    data: Any
+
+
+class YasError(ValueError):
+    pass
+
+
+# ── Serialization ──────────────────────────────────────────────────────────────
+
+
+def serialize_message(
+    data: Any, sender: str = "", purpose: int = MessagePurpose.NOTIFICATION
+) -> bytes:
+    sender_bytes = sender.encode("ascii")
+    header = (
+        YAS_HEADER
+        + struct.pack("<HHQ", purpose, get_data_type_id(data), len(sender_bytes))
+        + sender_bytes
+    )
+    return header + _serialize_payload(data)
+
+
+def _serialize_payload(data: Any) -> bytes:
+    if isinstance(data, FloatRingBuffer):
+        return (
+            YAS_HEADER
+            + struct.pack("<HQ", DataTypeId.FLOAT_RING_BUFFER, data.num_samples)
+            + data.samples
+            + struct.pack("<IQ", data.id, data.ts)
+        )
+    if isinstance(data, (dict, list)):
+        payload = json.dumps(data).encode("utf-8")
+        return YAS_HEADER + struct.pack("<Q", len(payload)) + payload
+    if isinstance(data, (TextData, str)):
+        text = data.text if isinstance(data, TextData) else data
+        payload = text.encode("utf-8")
+        return YAS_HEADER + struct.pack("<Q", len(payload)) + payload
+    if isinstance(data, BinaryData):
+        return data.data
+    if isinstance(data, (NullData, UndefinedData)) or data is None:
+        return b"\x00"
+    raise YasError(f"serialize_message: unsupported data type: {type(data)!r}")
+
+
+# ── Deserialization ────────────────────────────────────────────────────────────
+
+
+def deserialize_message(buffer: bytes | bytearray | memoryview) -> Message:
+    buf = bytes(buffer)
+    offset = _parse_yas_header(buf, 0)
+
+    if len(buf) < offset + 12:
+        raise YasError("deserialize_message: truncated message header")
+    purpose, data_type, sender_length = struct.unpack_from("<HHQ", buf, offset)
+    offset += 12
+
+    if len(buf) < offset + sender_length:
+        raise YasError("deserialize_message: truncated sender")
+    sender = buf[offset : offset + sender_length].decode("ascii", errors="replace")
+    offset += sender_length
+
+    return Message(purpose=purpose, sender=sender, data=_deserialize_payload(buf, offset, data_type))
+
+
+def _deserialize_payload(buf: bytes, offset: int, data_type: int) -> Any:
+    if data_type == DataTypeId.FLOAT_RING_BUFFER:
+        return _deserialize_ring_buffer(buf, offset)
+    if data_type == DataTypeId.JSON:
+        return json.loads(_deserialize_sized_text(buf, offset))
+    if data_type == DataTypeId.STRING:
+        return TextData(_deserialize_sized_text(buf, offset))
+    if data_type == DataTypeId.BINARY:
+        return BinaryData(buf[offset:])
+    if data_type == DataTypeId.NULL:
+        return NullData()
+    if data_type == DataTypeId.UNDEFINED:
+        return UndefinedData()
+    raise YasError(f"deserialize_message: unsupported data type id: {data_type}")
+
+
+def _deserialize_sized_text(buf: bytes, offset: int) -> str:
+    offset = _parse_yas_header(buf, offset)
+    if len(buf) < offset + 8:
+        raise YasError("deserialize_message: truncated text payload")
+    (length,) = struct.unpack_from("<Q", buf, offset)
+    offset += 8
+    if len(buf) < offset + length:
+        raise YasError("deserialize_message: truncated text payload body")
+    return buf[offset : offset + length].decode("utf-8")
+
+
+def _deserialize_ring_buffer(buf: bytes, offset: int) -> FloatRingBuffer:
+    offset = _parse_yas_header(buf, offset)
+
+    # The type-id field width differs between writers: the C++ runtime writes
+    # uint16, the TypeScript frontend writes uint32. Detect which dialect fits
+    # by checking that the implied sample count fills the frame exactly.
+    for type_width, type_fmt in ((2, "<H"), (4, "<I")):
+        pos = offset
+        if len(buf) < pos + type_width + 8:
+            continue
+        (type_id,) = struct.unpack_from(type_fmt, buf, pos)
+        if type_id != DataTypeId.FLOAT_RING_BUFFER:
+            continue
+        pos += type_width
+        (num_samples,) = struct.unpack_from("<Q", buf, pos)
+        pos += 8
+        if len(buf) != pos + num_samples * 4 + 4 + 8:
+            continue
+        samples = buf[pos : pos + num_samples * 4]
+        pos += num_samples * 4
+        rb_id, ts = struct.unpack_from("<IQ", buf, pos)
+        return FloatRingBuffer(samples=samples, id=rb_id, ts=ts)
+
+    raise YasError("deserialize_message: malformed FloatRingBuffer payload")
+
+
+def _parse_yas_header(buf: bytes, offset: int) -> int:
+    if len(buf) < offset + 7 or buf[offset : offset + 3] != b"yas":
+        raise YasError("Invalid YAS buffer header")
+    return offset + 7
+
+
+def is_yas_message(buffer: bytes | bytearray | memoryview) -> bool:
+    buf = bytes(buffer[:3])
+    return buf == b"yas"

package/bundle/hkp-rt/lib/CMakeLists.txt

@@ -0,0 +1,193 @@
+cmake_minimum_required(VERSION 3.15...3.31)
+project(hkp-rt-lib VERSION 1.0 LANGUAGES CXX)
+
+option(BUILD_MAC_INTEL "Build for intel architecture on Apple Silicon" OFF)
+
+if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
+endif()
+
+if(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
+  set(PLATFORM_SRC src/services/core_audio.cpp)
+
+  # Check if CMAKE_OSX_ARCHITECTURES is already set by parent (e.g., via arch -x86_64)
+  if(CMAKE_OSX_ARCHITECTURES)
+    message("Using parent-specified architecture: ${CMAKE_OSX_ARCHITECTURES}")
+  elseif(BUILD_MAC_INTEL)
+    message("Building for Intel architecture on Apple Silicon")
+    set(CMAKE_OSX_ARCHITECTURES "x86_64")
+  else()
+    message("Building for Apple Silicon")
+    set(CMAKE_OSX_ARCHITECTURES "arm64")
+  endif()
+endif()
+
+if(IOS)
+  # iOS: arm64 only; no desktop audio capture service (add mobile CoreAudio impl here if needed)
+  message(STATUS "hkp-rt-lib: building for iOS (arm64)")
+  set(CMAKE_OSX_ARCHITECTURES "arm64")
+endif()
+
+option(ENABLE_FFMPEG "Enable FFMPEG support" ON)
+
+if (AVCPP_FOUND AND ENABLE_FFMPEG)
+    set (PLATFORM_SRC ${PLATFORM_SRC} 
+          src/services/ffmpeg/ffmpeg.cpp 
+          src/services/ffmpeg/detail.cpp 
+    )
+    set (PLATFORM_DEFINES ${PLATFORM_DEFINES} "AVCPP_FOUND")
+
+    add_subdirectory(hkp-bundles) # bundles depend on ffmpeg
+endif()
+
+if(CMAKE_SYSTEM_NAME STREQUAL "Windows")
+endif()
+
+# ── mp4-to-wav service (fdk-aac + minimp4) ────────────────────────────────────
+# fdk-aac is found via find_package() in 3rdparty/CMakeLists.txt, where vcpkg's
+# CMAKE_PREFIX_PATH is active.  IMPORTED targets created there are globally
+# visible, so we check for the target here rather than re-running find_*.
+# Disabled on iOS: the iOS vcpkg manifest intentionally omits fdk-aac.
+if(NOT IOS)
+    if(VCPKG_INSTALLED_DIR AND VCPKG_TARGET_TRIPLET)
+        find_package(fdk-aac CONFIG QUIET
+            HINTS "${VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}"
+        )
+    endif()
+    if(TARGET FDK-AAC::fdk-aac)
+        message(STATUS "hkp-rt-lib: mp4-to-wav service enabled")
+        list(APPEND PLATFORM_SRC src/services/mp4_to_wav.cpp src/services/mp4_to_wav.h)
+        list(APPEND PLATFORM_DEFINES "HKP_MP4_TO_WAV_ENABLED")
+        set(_HKP_MP4_TO_WAV ON)
+    else()
+        message(STATUS "hkp-rt-lib: mp4-to-wav service disabled (fdk-aac not found; run ./3rdparty/vcpkg/vcpkg install then rm build/CMakeCache.txt)")
+    endif()
+endif()
+
+# ── mp3 encoding for audio-encode (LAME) ─────────────────────────────────────
+# Found the way fdk-aac is above. Without it, audio-encode still registers and
+# encodes wav; mp3 reports that it is not compiled in. macOS only for now: the
+# mobile manifests do not carry mp3lame.
+if(NOT IOS AND NOT ANDROID)
+    if(VCPKG_INSTALLED_DIR AND VCPKG_TARGET_TRIPLET)
+        find_package(mp3lame CONFIG QUIET
+            HINTS "${VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}"
+        )
+    endif()
+    if(TARGET mp3lame::mp3lame)
+        message(STATUS "hkp-rt-lib: audio-encode mp3 enabled (LAME)")
+        list(APPEND PLATFORM_DEFINES "HKP_MP3_ENABLED")
+        set(_HKP_MP3 ON)
+    else()
+        message(STATUS "hkp-rt-lib: audio-encode mp3 disabled (mp3lame not found; add it to the vcpkg manifest and rm build/CMakeCache.txt)")
+    endif()
+endif()
+
+add_library(${PROJECT_NAME} STATIC)
+target_sources(${PROJECT_NAME}
+    PRIVATE
+      src/common/websocket_server_session.cpp
+      src/runtime.cpp
+      src/assets.cpp
+      src/registry.cpp
+      src/service.cpp
+      src/sub_runtime.cpp
+      src/app.cpp
+      src/coordinator_links.cpp
+      src/discovery/discovery.cpp
+      src/http/auth.cpp
+      src/http/server.cpp
+      src/http/front_door.cpp
+      src/types/models.cpp 
+      src/types/message.cpp
+      src/types/data.cpp 
+      src/types/ringbuffer.cpp
+      src/common/websocket_server_listener.cpp      
+      src/common/websocket_client_session.cpp
+      src/common/link_socket.cpp
+      src/common/link_socket.h
+      #services
+      src/common/inja.h
+      src/common/websocket_server.cpp
+      src/services/static.cpp
+      src/services/http_client.cpp
+      src/services/http_server/http_server.cpp
+      src/services/http_server/http_server_subservices.cpp
+      src/services/http_server/http_server_impl.cpp
+      src/services/http_server/http_listener.cpp
+      src/services/http_server/http_session.cpp
+      src/services/http_server/http_stream_listener.cpp
+      src/services/http_client.cpp
+      src/services/text_generation.cpp
+      src/services/speech_to_text.cpp
+      src/services/text_to_speech.cpp
+      src/services/audio_encode.cpp
+      src/services/tts_inflect.cpp
+      src/services/websocket_writer.cpp
+      src/services/peer_server/peer_registry.cpp
+      src/services/peer_server/peer_ws_session.cpp
+      src/services/peer_server/peer_server_session.cpp
+      src/services/peer_server/peer_server_listener.cpp
+      src/services/websocket_server_service.h
+      src/services/filesystem.h
+      src/services/map.h
+      src/services/filter.h
+      src/services/cache.h
+      ${PLATFORM_SRC}
+)
+
+target_link_libraries(${PROJECT_NAME} PUBLIC hkp-rt-deps)
+target_compile_features(${PROJECT_NAME} PRIVATE cxx_std_20)
+set_target_properties(${PROJECT_NAME} PROPERTIES CXX_STANDARD 20 CXX_EXTENSIONS OFF CXX_STANDARD_REQUIRED ON)
+
+if(CMAKE_SYSTEM_NAME STREQUAL "Linux" AND NOT ANDROID)
+  target_compile_definitions(${PROJECT_NAME} PUBLIC "IS_LINUX" ${PLATFORM_DEFINES} CROW_USE_BOOST=1)
+endif()
+
+if(ANDROID)
+  # Android is POSIX/Linux-like: it reuses the getifaddrs + BSD-socket paths, so
+  # define IS_LINUX alongside IS_ANDROID. discoveryPlatformName() checks
+  # IS_ANDROID first and reports "android". No CoreFoundation/frameworks here;
+  # liblog provides __android_log_print for native logging.
+  target_compile_definitions(${PROJECT_NAME} PUBLIC "IS_ANDROID" "IS_LINUX" ${PLATFORM_DEFINES} CROW_USE_BOOST=1)
+  find_library(ANDROID_LOG_LIB log)
+  target_link_libraries(${PROJECT_NAME} PUBLIC ${ANDROID_LOG_LIB})
+endif()
+if(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
+  target_compile_definitions(${PROJECT_NAME} PUBLIC "IS_MACOS" ${PLATFORM_DEFINES} CROW_USE_BOOST=1)
+  find_library(COREFOUNDATION_FRAMEWORK CoreFoundation REQUIRED)
+  target_link_libraries(${PROJECT_NAME} PUBLIC ${COREFOUNDATION_FRAMEWORK})
+endif()
+
+if(CMAKE_SYSTEM_NAME STREQUAL "Windows")
+  target_compile_definitions(${PROJECT_NAME} PUBLIC "IS_WINDOWS" ${PLATFORM_DEFINES} CROW_USE_BOOST=1)
+  add_definitions(-D_WIN32_WINNT=0x0600)
+  target_compile_options(${PROJECT_NAME} PRIVATE /bigobj)
+  # ws2_32: Winsock; iphlpapi: GetAdaptersAddresses (LAN discovery interface enumeration).
+  target_link_libraries(${PROJECT_NAME} PUBLIC ws2_32 iphlpapi)
+endif()
+
+if(IOS)
+  target_compile_definitions(${PROJECT_NAME} PUBLIC "IS_IOS" "IS_APPLE" ${PLATFORM_DEFINES} CROW_USE_BOOST=1)
+  find_library(COREFOUNDATION_FRAMEWORK CoreFoundation REQUIRED)
+  target_link_libraries(${PROJECT_NAME} PUBLIC ${COREFOUNDATION_FRAMEWORK})
+endif()
+
+if(_HKP_MP3)
+  target_link_libraries(${PROJECT_NAME} PRIVATE mp3lame::mp3lame)
+endif()
+
+if(_HKP_MP4_TO_WAV)
+  # fdk-aac IMPORTED target carries its own include dirs and library path.
+  target_link_libraries(${PROJECT_NAME} PRIVATE FDK-AAC::fdk-aac)
+endif()
+
+target_include_directories(${PROJECT_NAME}
+    PRIVATE
+        # where the library itself will look for its internal headers
+        ${CMAKE_CURRENT_SOURCE_DIR}/src
+    PUBLIC
+        # where top-level project will look for the library's public headers
+        $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+        # where external projects will look for the library's public headers
+        $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
+)

package/bundle/hkp-rt/lib/hkp-bundles/CMakeLists.txt

@@ -0,0 +1,55 @@
+cmake_minimum_required(VERSION 3.15...3.31)
+project(hkp-rt-bundle VERSION 1.0 LANGUAGES CXX)
+
+
+set (SOURCES ../src/services/ffmpeg/ffmpeg.cpp 
+              ../src/services/ffmpeg/detail.cpp
+              plugin_register.cpp)
+
+add_library(${PROJECT_NAME} SHARED ${SOURCES})
+target_link_libraries(${PROJECT_NAME} hkp-rt-deps hkp-rt-lib)
+target_compile_features(${PROJECT_NAME} PRIVATE cxx_std_20)
+set_target_properties(${PROJECT_NAME} PROPERTIES CXX_STANDARD 20 CXX_EXTENSIONS OFF CXX_STANDARD_REQUIRED ON)
+
+if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
+  target_compile_definitions(${PROJECT_NAME} PUBLIC "IS_LINUX" ${PLATFORM_DEFINES} CROW_USE_BOOST=1)
+endif()
+if(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
+  target_compile_definitions(${PROJECT_NAME} PUBLIC "IS_MACOS" ${PLATFORM_DEFINES} CROW_USE_BOOST=1)
+endif()
+
+if(CMAKE_SYSTEM_NAME STREQUAL "Windows")
+  target_compile_definitions(${PROJECT_NAME} PUBLIC "IS_WINDOWS" ${PLATFORM_DEFINES} CROW_USE_BOOST=1)
+  add_definitions(-D_WIN32_WINNT=0x0600)
+  target_compile_options(${PROJECT_NAME} PRIVATE /bigobj)
+endif()
+
+target_include_directories(${PROJECT_NAME}
+    PRIVATE
+        ${CMAKE_CURRENT_SOURCE_DIR}/../src
+        ${CMAKE_CURRENT_SOURCE_DIR}/../include
+    PUBLIC
+        $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../include>
+        $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
+)
+
+# Add explicit code signing on macOS
+if(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
+    add_custom_command(TARGET ${PROJECT_NAME} POST_BUILD
+        COMMAND codesign --force --sign - $<TARGET_FILE:${PROJECT_NAME}>
+        COMMENT "Code signing ${PROJECT_NAME}"
+        VERBATIM
+    )
+endif()
+
+# Custom build rule to copy the bundle to ~/.hkp/bundles
+# This runs after code signing due to command order
+add_custom_command(TARGET ${PROJECT_NAME} POST_BUILD
+    COMMAND ${CMAKE_COMMAND} -E echo "Copying bundle to ~/.hkp/bundles"
+    COMMAND ${CMAKE_COMMAND} -E make_directory "$ENV{HOME}/.hkp/bundles"
+    COMMAND ${CMAKE_COMMAND} -E copy_if_different
+        $<TARGET_FILE:${PROJECT_NAME}>
+        "$ENV{HOME}/.hkp/bundles/$<TARGET_FILE_NAME:${PROJECT_NAME}>"
+    COMMENT "Copying ${PROJECT_NAME} bundle to ~/.hkp/bundles"
+    VERBATIM
+)

package/bundle/hkp-rt/lib/hkp-bundles/plugin_register.cpp

@@ -0,0 +1,14 @@
+#include "registry.h"
+#include "../src/services/ffmpeg/ffmpeg.h"
+
+// This function will be called by the host application to register services
+extern "C" void hkp_register_plugin(hkp::Registry* registry)
+{
+    // Register the Ffmpeg service from this plugin
+    registry->registerService<hkp::Ffmpeg>();
+}
+// This fuction will be called by the host application to introspect the plugin
+extern "C" const char* hkp_plugin_info()
+{
+    return R"({"type": "bundle", "services": ["ffmpeg"]})";
+}