@valon-technologies/gestalt 0.0.1-alpha.12 → 0.0.1-alpha.14

package/src/runtime-log-host.ts

@@ -0,0 +1,244 @@
+import { connect } from "node:net";
+import { Writable } from "node:stream";
+
+import type { MessageInitShape } from "@bufbuild/protobuf";
+import {
+  createClient,
+  type Client,
+  type Interceptor,
+} from "@connectrpc/connect";
+import { createGrpcTransport } from "@connectrpc/connect-node";
+
+import {
+  AppendPluginRuntimeLogsRequestSchema,
+  type AppendPluginRuntimeLogsResponse,
+  PluginRuntimeLogHost as PluginRuntimeLogHostService,
+  PluginRuntimeLogStream,
+} from "../gen/v1/pluginruntime_pb.ts";
+
+export const ENV_RUNTIME_LOG_HOST_SOCKET = "GESTALT_RUNTIME_LOG_SOCKET";
+export const ENV_RUNTIME_LOG_HOST_SOCKET_TOKEN = `${ENV_RUNTIME_LOG_HOST_SOCKET}_TOKEN`;
+export const ENV_RUNTIME_SESSION_ID = "GESTALT_RUNTIME_SESSION_ID";
+
+const RUNTIME_LOG_RELAY_TOKEN_HEADER = "x-gestalt-host-service-relay-token";
+
+export type RuntimeLogStreamName = "stdout" | "stderr" | "runtime";
+export type RuntimeLogStreamInput =
+  | RuntimeLogStreamName
+  | PluginRuntimeLogStream;
+export type RuntimeLogAppendLogsInput = MessageInitShape<
+  typeof AppendPluginRuntimeLogsRequestSchema
+>;
+export type RuntimeLogAppendResponseMessage = AppendPluginRuntimeLogsResponse;
+
+export interface RuntimeLogAppendInput {
+  sessionId?: string;
+  message: string | Uint8Array;
+  stream?: RuntimeLogStreamInput;
+  observedAt?: Date;
+  sourceSeq?: number | bigint;
+}
+
+export interface RuntimeLogWriterOptions {
+  sessionId?: string;
+  stream?: RuntimeLogStreamInput;
+  sourceSeqStart?: number | bigint;
+}
+
+export class RuntimeLogHost {
+  private readonly client: Client<typeof PluginRuntimeLogHostService>;
+  private sourceSeq = 0n;
+
+  constructor() {
+    const target = process.env[ENV_RUNTIME_LOG_HOST_SOCKET];
+    if (!target) {
+      throw new Error(
+        `runtime log host: ${ENV_RUNTIME_LOG_HOST_SOCKET} is not set`,
+      );
+    }
+    const relayToken =
+      process.env[ENV_RUNTIME_LOG_HOST_SOCKET_TOKEN]?.trim() ?? "";
+    const transportOptions = runtimeLogTransportOptions(target);
+    const transport = createGrpcTransport({
+      ...transportOptions,
+      ...(transportOptions.nodeOptions
+        ? {
+            nodeOptions: {
+              createConnection: () =>
+                connect(transportOptions.nodeOptions!.path),
+            },
+          }
+        : {}),
+      interceptors: relayToken
+        ? [runtimeLogRelayTokenInterceptor(relayToken)]
+        : [],
+    });
+    this.client = createClient(PluginRuntimeLogHostService, transport);
+  }
+
+  async appendLogs(
+    request: RuntimeLogAppendLogsInput,
+  ): Promise<RuntimeLogAppendResponseMessage> {
+    return await this.client.appendLogs(request);
+  }
+
+  async append(
+    input: RuntimeLogAppendInput,
+  ): Promise<RuntimeLogAppendResponseMessage> {
+    const sourceSeq =
+      input.sourceSeq === undefined
+        ? (this.sourceSeq += 1n)
+        : BigInt(input.sourceSeq);
+    if (sourceSeq > this.sourceSeq) {
+      this.sourceSeq = sourceSeq;
+    }
+    return await this.appendLogs({
+      sessionId: runtimeSessionId(input.sessionId),
+      logs: [
+        {
+          stream: runtimeLogStream(input.stream ?? "runtime"),
+          message: runtimeLogMessage(input.message),
+          observedAt: toProtoTimestamp(input.observedAt ?? new Date()),
+          sourceSeq,
+        },
+      ],
+    });
+  }
+
+  writer(options?: RuntimeLogWriterOptions): Writable;
+  writer(sessionId: string, options?: RuntimeLogWriterOptions): Writable;
+  writer(
+    sessionIdOrOptions: string | RuntimeLogWriterOptions = {},
+    options: RuntimeLogWriterOptions = {},
+  ): Writable {
+    const writerOptions =
+      typeof sessionIdOrOptions === "string"
+        ? options
+        : sessionIdOrOptions;
+    const sessionId = runtimeSessionId(
+      typeof sessionIdOrOptions === "string"
+        ? sessionIdOrOptions
+        : writerOptions.sessionId,
+    );
+    const stream = writerOptions.stream ?? "stdout";
+    let sourceSeq = BigInt(writerOptions.sourceSeqStart ?? 0);
+
+    return new Writable({
+      write: (chunk: Buffer | string, encoding, callback) => {
+        const actualEncoding = (
+          String(encoding) === "buffer" ? "utf8" : encoding
+        ) as BufferEncoding;
+        const message =
+          typeof chunk === "string"
+            ? chunk
+            : Buffer.from(chunk).toString(actualEncoding);
+        sourceSeq += 1n;
+        this.append({
+          sessionId,
+          stream,
+          message,
+          sourceSeq,
+        }).then(
+          () => callback(),
+          (error: unknown) => callback(toError(error)),
+        );
+      },
+    });
+  }
+}
+
+function runtimeSessionId(sessionId?: string): string {
+  const value = (sessionId ?? process.env[ENV_RUNTIME_SESSION_ID] ?? "").trim();
+  if (!value) {
+    throw new Error(`runtime session: ${ENV_RUNTIME_SESSION_ID} is not set`);
+  }
+  return value;
+}
+
+function runtimeLogTransportOptions(rawTarget: string): {
+  baseUrl: string;
+  nodeOptions?: { path: string };
+} {
+  const target = rawTarget.trim();
+  if (!target) {
+    throw new Error("runtime log host: transport target is required");
+  }
+  if (target.startsWith("tcp://")) {
+    const address = target.slice("tcp://".length).trim();
+    if (!address) {
+      throw new Error(
+        `runtime log host: tcp target ${JSON.stringify(rawTarget)} is missing host:port`,
+      );
+    }
+    return { baseUrl: `http://${address}` };
+  }
+  if (target.startsWith("tls://")) {
+    const address = target.slice("tls://".length).trim();
+    if (!address) {
+      throw new Error(
+        `runtime log host: tls target ${JSON.stringify(rawTarget)} is missing host:port`,
+      );
+    }
+    return { baseUrl: `https://${address}` };
+  }
+  if (target.startsWith("unix://")) {
+    const socketPath = target.slice("unix://".length).trim();
+    if (!socketPath) {
+      throw new Error(
+        `runtime log host: unix target ${JSON.stringify(rawTarget)} is missing a socket path`,
+      );
+    }
+    return { baseUrl: "http://localhost", nodeOptions: { path: socketPath } };
+  }
+  if (target.includes("://")) {
+    const parsed = new URL(target);
+    throw new Error(
+      `runtime log host: unsupported target scheme ${JSON.stringify(parsed.protocol.replace(/:$/, ""))}`,
+    );
+  }
+  return { baseUrl: "http://localhost", nodeOptions: { path: target } };
+}
+
+function runtimeLogRelayTokenInterceptor(token: string): Interceptor {
+  return (next) => async (req) => {
+    req.header.set(RUNTIME_LOG_RELAY_TOKEN_HEADER, token);
+    return await next(req);
+  };
+}
+
+function runtimeLogStream(stream: RuntimeLogStreamInput): PluginRuntimeLogStream {
+  if (typeof stream === "number") {
+    return stream;
+  }
+  switch (stream.trim().toLowerCase()) {
+    case "stdout":
+      return PluginRuntimeLogStream.STDOUT;
+    case "stderr":
+      return PluginRuntimeLogStream.STDERR;
+    case "runtime":
+      return PluginRuntimeLogStream.RUNTIME;
+    default:
+      throw new Error(`unsupported runtime log stream ${JSON.stringify(stream)}`);
+  }
+}
+
+function runtimeLogMessage(message: string | Uint8Array): string {
+  if (typeof message === "string") {
+    return message;
+  }
+  return Buffer.from(message).toString("utf8");
+}
+
+function toProtoTimestamp(value: Date): { seconds: bigint; nanos: number } {
+  const millis = value.getTime();
+  const seconds = Math.floor(millis / 1000);
+  const nanos = Math.trunc((millis - (seconds * 1000)) * 1_000_000);
+  return {
+    seconds: BigInt(seconds),
+    nanos,
+  };
+}
+
+function toError(error: unknown): Error {
+  return error instanceof Error ? error : new Error(String(error));
+}

package/src/runtime.ts

@@ -1,4 +1,4 @@
-import { existsSync, rmSync, writeFileSync } from "node:fs";
+import { rmSync, writeFileSync } from "node:fs";
 import { createServer } from "node:http2";
 import { dirname, resolve } from "node:path";
 
@@ -12,6 +12,9 @@ import {
 } from "@connectrpc/connect";
 import { connectNodeAdapter } from "@connectrpc/connect-node";
 
+import {
+  AgentProvider as AgentProviderService,
+} from "../gen/v1/agent_pb.ts";
 import {
   AuthenticationProvider as AuthenticationProviderService,
   AuthSessionSettingsSchema,
@@ -40,10 +43,12 @@ import {
   CatalogSchema as ProtoCatalogSchema,
   ConnectionMode as ProviderConnectionMode,
   GetSessionCatalogResponseSchema,
+  PostConnectResponseSchema,
   ResolveHTTPSubjectResponseSchema,
   OperationResultSchema,
   ProviderMetadataSchema,
   type HTTPSubjectRequest as ProtoHTTPSubjectRequest,
+  type PostConnectCredential as ProtoPostConnectCredential,
   type RequestContext as ProtoRequestContext,
   type ResolveHTTPSubjectRequest as ProtoResolveHTTPSubjectRequest,
   IntegrationProvider as IntegrationProviderService,
@@ -52,17 +57,26 @@ import {
   type GetSessionCatalogRequest,
   type StartProviderRequest,
 } from "../gen/v1/plugin_pb.ts";
+import {
+  PluginRuntimeProvider as PluginRuntimeProviderService,
+} from "../gen/v1/pluginruntime_pb.ts";
 import {
   ConfigureProviderResponseSchema,
   HealthCheckResponseSchema,
   ProviderIdentitySchema,
   ProviderKind as ProtoProviderKind,
   ProviderLifecycle,
+  StartRuntimeProviderResponseSchema,
   type ConfigureProviderRequest,
 } from "../gen/v1/runtime_pb.ts";
 import { S3 as S3Service } from "../gen/v1/s3_pb.ts";
 import { WorkflowProvider as WorkflowProviderService } from "../gen/v1/workflow_pb.ts";
 import { errorMessage, type Request } from "./api.ts";
+import {
+  AgentProvider,
+  createAgentProviderService,
+  isAgentProvider,
+} from "./agent.ts";
 import {
   AuthenticationProvider,
   isAuthenticationProvider,
@@ -77,11 +91,17 @@ import {
   type HTTPSubjectResolutionContext,
 } from "./http-subject.ts";
 import {
+  type ConnectedToken,
   PluginProvider,
   connectionModeToProtoValue,
   connectionParamToProto,
   isPluginProvider,
 } from "./plugin.ts";
+import {
+  PluginRuntimeProvider,
+  createPluginRuntimeProviderService,
+  isPluginRuntimeProvider,
+} from "./pluginruntime.ts";
 import {
   providerKindLabel,
   resolveDefaultProviderExport,
@@ -114,11 +134,6 @@ export const ENV_PROVIDER_PARENT_PID = "GESTALT_PLUGIN_PARENT_PID";
  * Environment variable used to request static catalog generation.
  */
 export const ENV_WRITE_CATALOG = "GESTALT_PLUGIN_WRITE_CATALOG";
-/**
- * Environment variable used to request generated manifest metadata export.
- */
-export const ENV_WRITE_MANIFEST_METADATA =
-  "GESTALT_PLUGIN_WRITE_MANIFEST_METADATA";
 /**
  * Protocol version currently implemented by the TypeScript runtime.
  */
@@ -127,6 +142,8 @@ export const CURRENT_PROTOCOL_VERSION = 3;
  * Command-line usage for the runtime entrypoint.
  */
 export const USAGE = "usage: bun run runtime.ts ROOT PROVIDER_TARGET";
+export { createAgentProviderService } from "./agent.ts";
+export { createPluginRuntimeProviderService } from "./pluginruntime.ts";
 export { createWorkflowProviderService } from "./workflow.ts";
 
 /**
@@ -146,6 +163,8 @@ export type LoadedProvider =
   | CacheProvider
   | SecretsProvider
   | S3Provider
+  | PluginRuntimeProvider
+  | AgentProvider
   | WorkflowProvider;
 
 type ProviderRuntimeEntry = {
@@ -205,6 +224,27 @@ const PROVIDER_RUNTIME_ENTRIES: Partial<
       router.service(S3Service, createS3Service(provider as S3Provider));
     },
   },
+  runtime: {
+    isProvider:
+      isPluginRuntimeProvider as (value: unknown) => value is LoadedProvider,
+    protoKind: ProtoProviderKind.RUNTIME,
+    registerService(router, provider) {
+      router.service(
+        PluginRuntimeProviderService,
+        createPluginRuntimeProviderService(provider as PluginRuntimeProvider),
+      );
+    },
+  },
+  agent: {
+    isProvider: isAgentProvider as (value: unknown) => value is LoadedProvider,
+    protoKind: ProtoProviderKind.AGENT,
+    registerService(router, provider) {
+      router.service(
+        AgentProviderService,
+        createAgentProviderService(provider as AgentProvider),
+      );
+    },
+  },
   workflow: {
     isProvider: isWorkflowProvider as (value: unknown) => value is LoadedProvider,
     protoKind: ProtoProviderKind.WORKFLOW,
@@ -301,19 +341,11 @@ export async function runLoadedProvider(
   }
 
   const catalogPath = process.env[ENV_WRITE_CATALOG];
-  const manifestMetadataPath = process.env[ENV_WRITE_MANIFEST_METADATA];
-  if (catalogPath || manifestMetadataPath) {
+  if (catalogPath) {
     if (!isPluginProvider(provider)) {
-      throw new Error(
-        "static catalog and manifest metadata generation are only supported for plugin providers",
-      );
-    }
-    if (catalogPath) {
-      writeFileSync(catalogPath, catalogToYaml(provider.staticCatalog()), "utf8");
-    }
-    if (manifestMetadataPath && provider.supportsManifestMetadata()) {
-      provider.writeManifestMetadata(manifestMetadataPath);
+      throw new Error("static catalog generation is only supported for plugin providers");
     }
+    writeFileSync(catalogPath, catalogToYaml(provider.staticCatalog()), "utf8");
     return;
   }
 
@@ -341,9 +373,7 @@ export async function serve(provider: LoadedProvider): Promise<void> {
   if (!socketPath) {
     throw new Error(`${ENV_PROVIDER_SOCKET} is required`);
   }
-  if (existsSync(socketPath)) {
-    rmSync(socketPath);
-  }
+  rmSync(socketPath, { force: true });
 
   const handler = connectNodeAdapter({
     grpc: true,
@@ -370,9 +400,7 @@ export async function serve(provider: LoadedProvider): Promise<void> {
             server.close(() => resolveClose());
           });
         } finally {
-          if (existsSync(socketPath)) {
-            rmSync(socketPath);
-          }
+          rmSync(socketPath, { force: true });
         }
       }
     })();
@@ -412,16 +440,20 @@ export function createRuntimeService(
 ): Partial<ServiceImpl<typeof ProviderLifecycle>> {
   return {
     async getProviderIdentity() {
-      return create(ProviderIdentitySchema, {
-        kind: providerRuntimeEntry(provider.kind).protoKind,
-        name: provider.name,
-        displayName: provider.displayName,
-        description: provider.description,
-        version: provider.version,
-        warnings: await provider.warnings(),
-        minProtocolVersion: CURRENT_PROTOCOL_VERSION,
-        maxProtocolVersion: CURRENT_PROTOCOL_VERSION,
-      });
+      try {
+        return create(ProviderIdentitySchema, {
+          kind: providerRuntimeEntry(provider.kind).protoKind,
+          name: provider.name,
+          displayName: provider.displayName,
+          description: provider.description,
+          version: provider.version,
+          warnings: await provider.warnings(),
+          minProtocolVersion: CURRENT_PROTOCOL_VERSION,
+          maxProtocolVersion: CURRENT_PROTOCOL_VERSION,
+        });
+      } catch (error) {
+        throw providerRuntimeError("provider identity", error);
+      }
     },
     async configureProvider(request: ConfigureProviderRequest) {
       assertProtocolVersion(request.protocolVersion);
@@ -431,10 +463,7 @@ export function createRuntimeService(
           objectFromUnknown(request.config),
         );
       } catch (error) {
-        throw new ConnectError(
-          `configure provider: ${errorMessage(error)}`,
-          Code.Unknown,
-        );
+        throw providerRuntimeError("configure provider", error);
       }
       return create(ConfigureProviderResponseSchema, {
         protocolVersion: CURRENT_PROTOCOL_VERSION,
@@ -458,6 +487,16 @@ export function createRuntimeService(
         });
       }
     },
+    async startProvider() {
+      try {
+        await provider.startProvider();
+      } catch (error) {
+        throw providerRuntimeError("start provider", error);
+      }
+      return create(StartRuntimeProviderResponseSchema, {
+        protocolVersion: CURRENT_PROTOCOL_VERSION,
+      });
+    },
   };
 }
 
@@ -473,27 +512,31 @@ export function createProviderService(
     throw new Error("provider is not a plugin provider");
   }
   return {
-    getMetadata() {
-      return create(ProviderMetadataSchema, {
-        name: provider.name,
-        displayName: provider.displayName,
-        description: provider.description,
-        connectionMode: connectionModeToProtoValue(
-          provider.connectionMode,
-        ) as ProviderConnectionMode,
-        authTypes: [...provider.authTypes],
-        connectionParams: Object.fromEntries(
-          Object.entries(provider.connectionParams).map(([key, value]) => [
-            key,
-            connectionParamToProto(value),
-          ]),
-        ),
-        staticCatalog: catalogToProto(provider.staticCatalog()),
-        supportsSessionCatalog: provider.supportsSessionCatalog(),
-        supportsPostConnect: false,
-        minProtocolVersion: CURRENT_PROTOCOL_VERSION,
-        maxProtocolVersion: CURRENT_PROTOCOL_VERSION,
-      });
+    async getMetadata() {
+      try {
+        return create(ProviderMetadataSchema, {
+          name: provider.name,
+          displayName: provider.displayName,
+          description: provider.description,
+          connectionMode: connectionModeToProtoValue(
+            provider.connectionMode,
+          ) as ProviderConnectionMode,
+          authTypes: [...provider.authTypes],
+          connectionParams: Object.fromEntries(
+            Object.entries(provider.connectionParams).map(([key, value]) => [
+              key,
+              connectionParamToProto(value),
+            ]),
+          ),
+          staticCatalog: catalogToProto(provider.staticCatalog()),
+          supportsSessionCatalog: provider.supportsSessionCatalog(),
+          supportsPostConnect: provider.supportsPostConnect(),
+          minProtocolVersion: CURRENT_PROTOCOL_VERSION,
+          maxProtocolVersion: CURRENT_PROTOCOL_VERSION,
+        });
+      } catch (error) {
+        throw providerRuntimeError("provider metadata", error);
+      }
     },
     async startProvider(request: StartProviderRequest) {
       assertProtocolVersion(request.protocolVersion);
@@ -503,10 +546,7 @@ export function createProviderService(
           objectFromUnknown(request.config),
         );
       } catch (error) {
-        throw new ConnectError(
-          `configure provider: ${errorMessage(error)}`,
-          Code.Unknown,
-        );
+        throw providerRuntimeError("configure provider", error);
       }
       return create(StartProviderResponseSchema, {
         protocolVersion: CURRENT_PROTOCOL_VERSION,
@@ -523,6 +563,7 @@ export function createProviderService(
             request.connectionParams,
             request.context,
             request.invocationToken,
+            request.idempotencyKey,
           ),
         ),
       );
@@ -583,11 +624,29 @@ export function createProviderService(
         catalog: catalogToProto(catalog),
       });
     },
-    async postConnect() {
-      throw new ConnectError(
-        "provider does not support post connect",
-        Code.Unimplemented,
-      );
+    async postConnect(request) {
+      if (!provider.supportsPostConnect()) {
+        throw new ConnectError(
+          "provider does not support post connect",
+          Code.Unimplemented,
+        );
+      }
+      let metadata: Record<string, string> | null | undefined;
+      try {
+        metadata = await provider.postConnectMetadata(
+          providerConnectedToken(request.token),
+        );
+      } catch (error) {
+        throw new ConnectError(
+          `post connect: ${errorMessage(error)}`,
+          Code.Unknown,
+        );
+      }
+      return create(PostConnectResponseSchema, {
+        metadata: {
+          ...(metadata ?? {}),
+        },
+      });
     },
   };
 }
@@ -758,6 +817,7 @@ function providerRequest(
   connectionParams: Record<string, string>,
   requestContext?: ProtoRequestContext,
   invocationToken = "",
+  idempotencyKey = "",
 ): Request {
   const subject = requestContext?.subject;
   const credential = requestContext?.credential;
@@ -787,6 +847,7 @@ function providerRequest(
       ...(requestContext?.workflow ?? {}),
     },
     invocationToken,
+    idempotencyKey: idempotencyKey.trim(),
   };
 }
 
@@ -822,6 +883,29 @@ function providerHTTPSubjectResolutionContext(
   };
 }
 
+function providerConnectedToken(
+  token?: ProtoPostConnectCredential,
+): ConnectedToken {
+  const metadataJson = token?.metadataJson ?? "";
+  return {
+    id: token?.id ?? "",
+    subjectId: token?.subjectId ?? "",
+    integration: token?.integration ?? "",
+    connection: token?.connection ?? "",
+    instance: token?.instance ?? "",
+    accessToken: token?.accessToken ?? "",
+    refreshToken: token?.refreshToken ?? "",
+    scopes: token?.scopes ?? "",
+    expiresAt: timestampToDate(token?.expiresAt),
+    lastRefreshedAt: timestampToDate(token?.lastRefreshedAt),
+    refreshErrorCount: token?.refreshErrorCount ?? 0,
+    metadataJson,
+    metadata: stringRecordFromJSON(metadataJson),
+    createdAt: timestampToDate(token?.createdAt),
+    updatedAt: timestampToDate(token?.updatedAt),
+  };
+}
+
 function providerStringLists(
   input: Record<string, { values?: string[] }> | undefined,
 ): Record<string, string[]> {
@@ -844,6 +928,10 @@ function providerRuntimeEntry(
   return entry;
 }
 
+function providerRuntimeError(label: string, error: unknown): ConnectError {
+  return new ConnectError(`${label}: ${errorMessage(error)}`, Code.Unknown);
+}
+
 function resolveLoadedProvider(
   candidate: unknown,
   kind: ProviderKind,
@@ -930,6 +1018,45 @@ function normalizeBigInt(value: number | bigint): bigint {
   return BigInt(Math.max(0, Math.trunc(value)));
 }
 
+function timestampToDate(
+  value: { seconds: bigint; nanos: number } | undefined,
+): Date | undefined {
+  if (!value) {
+    return undefined;
+  }
+  const seconds = Number(value.seconds ?? 0n);
+  const nanos = Number(value.nanos ?? 0);
+  if (!Number.isFinite(seconds) || !Number.isFinite(nanos)) {
+    return undefined;
+  }
+  const millis = (seconds * 1000) + Math.trunc(nanos / 1_000_000);
+  if (!Number.isFinite(millis)) {
+    return undefined;
+  }
+  return new Date(millis);
+}
+
+function stringRecordFromJSON(value: string): Record<string, string> {
+  if (!value.trim()) {
+    return {};
+  }
+  try {
+    const parsed = JSON.parse(value) as unknown;
+    if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
+      return {};
+    }
+    const output: Record<string, string> = {};
+    for (const [key, entry] of Object.entries(parsed)) {
+      if (typeof entry === "string") {
+        output[key] = entry;
+      }
+    }
+    return output;
+  } catch {
+    return {};
+  }
+}
+
 function cloneUint8Array(value: Uint8Array | undefined): Uint8Array {
   if (!value) {
     return new Uint8Array();