@valon-technologies/gestalt 0.0.1-alpha.13 → 0.0.1-alpha.14

package/src/runtime-log-host.ts

@@ -0,0 +1,244 @@
+import { connect } from "node:net";
+import { Writable } from "node:stream";
+
+import type { MessageInitShape } from "@bufbuild/protobuf";
+import {
+  createClient,
+  type Client,
+  type Interceptor,
+} from "@connectrpc/connect";
+import { createGrpcTransport } from "@connectrpc/connect-node";
+
+import {
+  AppendPluginRuntimeLogsRequestSchema,
+  type AppendPluginRuntimeLogsResponse,
+  PluginRuntimeLogHost as PluginRuntimeLogHostService,
+  PluginRuntimeLogStream,
+} from "../gen/v1/pluginruntime_pb.ts";
+
+export const ENV_RUNTIME_LOG_HOST_SOCKET = "GESTALT_RUNTIME_LOG_SOCKET";
+export const ENV_RUNTIME_LOG_HOST_SOCKET_TOKEN = `${ENV_RUNTIME_LOG_HOST_SOCKET}_TOKEN`;
+export const ENV_RUNTIME_SESSION_ID = "GESTALT_RUNTIME_SESSION_ID";
+
+const RUNTIME_LOG_RELAY_TOKEN_HEADER = "x-gestalt-host-service-relay-token";
+
+export type RuntimeLogStreamName = "stdout" | "stderr" | "runtime";
+export type RuntimeLogStreamInput =
+  | RuntimeLogStreamName
+  | PluginRuntimeLogStream;
+export type RuntimeLogAppendLogsInput = MessageInitShape<
+  typeof AppendPluginRuntimeLogsRequestSchema
+>;
+export type RuntimeLogAppendResponseMessage = AppendPluginRuntimeLogsResponse;
+
+export interface RuntimeLogAppendInput {
+  sessionId?: string;
+  message: string | Uint8Array;
+  stream?: RuntimeLogStreamInput;
+  observedAt?: Date;
+  sourceSeq?: number | bigint;
+}
+
+export interface RuntimeLogWriterOptions {
+  sessionId?: string;
+  stream?: RuntimeLogStreamInput;
+  sourceSeqStart?: number | bigint;
+}
+
+export class RuntimeLogHost {
+  private readonly client: Client<typeof PluginRuntimeLogHostService>;
+  private sourceSeq = 0n;
+
+  constructor() {
+    const target = process.env[ENV_RUNTIME_LOG_HOST_SOCKET];
+    if (!target) {
+      throw new Error(
+        `runtime log host: ${ENV_RUNTIME_LOG_HOST_SOCKET} is not set`,
+      );
+    }
+    const relayToken =
+      process.env[ENV_RUNTIME_LOG_HOST_SOCKET_TOKEN]?.trim() ?? "";
+    const transportOptions = runtimeLogTransportOptions(target);
+    const transport = createGrpcTransport({
+      ...transportOptions,
+      ...(transportOptions.nodeOptions
+        ? {
+            nodeOptions: {
+              createConnection: () =>
+                connect(transportOptions.nodeOptions!.path),
+            },
+          }
+        : {}),
+      interceptors: relayToken
+        ? [runtimeLogRelayTokenInterceptor(relayToken)]
+        : [],
+    });
+    this.client = createClient(PluginRuntimeLogHostService, transport);
+  }
+
+  async appendLogs(
+    request: RuntimeLogAppendLogsInput,
+  ): Promise<RuntimeLogAppendResponseMessage> {
+    return await this.client.appendLogs(request);
+  }
+
+  async append(
+    input: RuntimeLogAppendInput,
+  ): Promise<RuntimeLogAppendResponseMessage> {
+    const sourceSeq =
+      input.sourceSeq === undefined
+        ? (this.sourceSeq += 1n)
+        : BigInt(input.sourceSeq);
+    if (sourceSeq > this.sourceSeq) {
+      this.sourceSeq = sourceSeq;
+    }
+    return await this.appendLogs({
+      sessionId: runtimeSessionId(input.sessionId),
+      logs: [
+        {
+          stream: runtimeLogStream(input.stream ?? "runtime"),
+          message: runtimeLogMessage(input.message),
+          observedAt: toProtoTimestamp(input.observedAt ?? new Date()),
+          sourceSeq,
+        },
+      ],
+    });
+  }
+
+  writer(options?: RuntimeLogWriterOptions): Writable;
+  writer(sessionId: string, options?: RuntimeLogWriterOptions): Writable;
+  writer(
+    sessionIdOrOptions: string | RuntimeLogWriterOptions = {},
+    options: RuntimeLogWriterOptions = {},
+  ): Writable {
+    const writerOptions =
+      typeof sessionIdOrOptions === "string"
+        ? options
+        : sessionIdOrOptions;
+    const sessionId = runtimeSessionId(
+      typeof sessionIdOrOptions === "string"
+        ? sessionIdOrOptions
+        : writerOptions.sessionId,
+    );
+    const stream = writerOptions.stream ?? "stdout";
+    let sourceSeq = BigInt(writerOptions.sourceSeqStart ?? 0);
+
+    return new Writable({
+      write: (chunk: Buffer | string, encoding, callback) => {
+        const actualEncoding = (
+          String(encoding) === "buffer" ? "utf8" : encoding
+        ) as BufferEncoding;
+        const message =
+          typeof chunk === "string"
+            ? chunk
+            : Buffer.from(chunk).toString(actualEncoding);
+        sourceSeq += 1n;
+        this.append({
+          sessionId,
+          stream,
+          message,
+          sourceSeq,
+        }).then(
+          () => callback(),
+          (error: unknown) => callback(toError(error)),
+        );
+      },
+    });
+  }
+}
+
+function runtimeSessionId(sessionId?: string): string {
+  const value = (sessionId ?? process.env[ENV_RUNTIME_SESSION_ID] ?? "").trim();
+  if (!value) {
+    throw new Error(`runtime session: ${ENV_RUNTIME_SESSION_ID} is not set`);
+  }
+  return value;
+}
+
+function runtimeLogTransportOptions(rawTarget: string): {
+  baseUrl: string;
+  nodeOptions?: { path: string };
+} {
+  const target = rawTarget.trim();
+  if (!target) {
+    throw new Error("runtime log host: transport target is required");
+  }
+  if (target.startsWith("tcp://")) {
+    const address = target.slice("tcp://".length).trim();
+    if (!address) {
+      throw new Error(
+        `runtime log host: tcp target ${JSON.stringify(rawTarget)} is missing host:port`,
+      );
+    }
+    return { baseUrl: `http://${address}` };
+  }
+  if (target.startsWith("tls://")) {
+    const address = target.slice("tls://".length).trim();
+    if (!address) {
+      throw new Error(
+        `runtime log host: tls target ${JSON.stringify(rawTarget)} is missing host:port`,
+      );
+    }
+    return { baseUrl: `https://${address}` };
+  }
+  if (target.startsWith("unix://")) {
+    const socketPath = target.slice("unix://".length).trim();
+    if (!socketPath) {
+      throw new Error(
+        `runtime log host: unix target ${JSON.stringify(rawTarget)} is missing a socket path`,
+      );
+    }
+    return { baseUrl: "http://localhost", nodeOptions: { path: socketPath } };
+  }
+  if (target.includes("://")) {
+    const parsed = new URL(target);
+    throw new Error(
+      `runtime log host: unsupported target scheme ${JSON.stringify(parsed.protocol.replace(/:$/, ""))}`,
+    );
+  }
+  return { baseUrl: "http://localhost", nodeOptions: { path: target } };
+}
+
+function runtimeLogRelayTokenInterceptor(token: string): Interceptor {
+  return (next) => async (req) => {
+    req.header.set(RUNTIME_LOG_RELAY_TOKEN_HEADER, token);
+    return await next(req);
+  };
+}
+
+function runtimeLogStream(stream: RuntimeLogStreamInput): PluginRuntimeLogStream {
+  if (typeof stream === "number") {
+    return stream;
+  }
+  switch (stream.trim().toLowerCase()) {
+    case "stdout":
+      return PluginRuntimeLogStream.STDOUT;
+    case "stderr":
+      return PluginRuntimeLogStream.STDERR;
+    case "runtime":
+      return PluginRuntimeLogStream.RUNTIME;
+    default:
+      throw new Error(`unsupported runtime log stream ${JSON.stringify(stream)}`);
+  }
+}
+
+function runtimeLogMessage(message: string | Uint8Array): string {
+  if (typeof message === "string") {
+    return message;
+  }
+  return Buffer.from(message).toString("utf8");
+}
+
+function toProtoTimestamp(value: Date): { seconds: bigint; nanos: number } {
+  const millis = value.getTime();
+  const seconds = Math.floor(millis / 1000);
+  const nanos = Math.trunc((millis - (seconds * 1000)) * 1_000_000);
+  return {
+    seconds: BigInt(seconds),
+    nanos,
+  };
+}
+
+function toError(error: unknown): Error {
+  return error instanceof Error ? error : new Error(String(error));
+}

package/src/runtime.ts

@@ -1,4 +1,4 @@
-import { existsSync, rmSync, writeFileSync } from "node:fs";
+import { rmSync, writeFileSync } from "node:fs";
 import { createServer } from "node:http2";
 import { dirname, resolve } from "node:path";
 
@@ -48,7 +48,7 @@ import {
   OperationResultSchema,
   ProviderMetadataSchema,
   type HTTPSubjectRequest as ProtoHTTPSubjectRequest,
-  type IntegrationToken as ProtoIntegrationToken,
+  type PostConnectCredential as ProtoPostConnectCredential,
   type RequestContext as ProtoRequestContext,
   type ResolveHTTPSubjectRequest as ProtoResolveHTTPSubjectRequest,
   IntegrationProvider as IntegrationProviderService,
@@ -57,12 +57,16 @@ import {
   type GetSessionCatalogRequest,
   type StartProviderRequest,
 } from "../gen/v1/plugin_pb.ts";
+import {
+  PluginRuntimeProvider as PluginRuntimeProviderService,
+} from "../gen/v1/pluginruntime_pb.ts";
 import {
   ConfigureProviderResponseSchema,
   HealthCheckResponseSchema,
   ProviderIdentitySchema,
   ProviderKind as ProtoProviderKind,
   ProviderLifecycle,
+  StartRuntimeProviderResponseSchema,
   type ConfigureProviderRequest,
 } from "../gen/v1/runtime_pb.ts";
 import { S3 as S3Service } from "../gen/v1/s3_pb.ts";
@@ -93,6 +97,11 @@ import {
   connectionParamToProto,
   isPluginProvider,
 } from "./plugin.ts";
+import {
+  PluginRuntimeProvider,
+  createPluginRuntimeProviderService,
+  isPluginRuntimeProvider,
+} from "./pluginruntime.ts";
 import {
   providerKindLabel,
   resolveDefaultProviderExport,
@@ -125,11 +134,6 @@ export const ENV_PROVIDER_PARENT_PID = "GESTALT_PLUGIN_PARENT_PID";
  * Environment variable used to request static catalog generation.
  */
 export const ENV_WRITE_CATALOG = "GESTALT_PLUGIN_WRITE_CATALOG";
-/**
- * Environment variable used to request generated manifest metadata export.
- */
-export const ENV_WRITE_MANIFEST_METADATA =
-  "GESTALT_PLUGIN_WRITE_MANIFEST_METADATA";
 /**
  * Protocol version currently implemented by the TypeScript runtime.
  */
@@ -139,6 +143,7 @@ export const CURRENT_PROTOCOL_VERSION = 3;
  */
 export const USAGE = "usage: bun run runtime.ts ROOT PROVIDER_TARGET";
 export { createAgentProviderService } from "./agent.ts";
+export { createPluginRuntimeProviderService } from "./pluginruntime.ts";
 export { createWorkflowProviderService } from "./workflow.ts";
 
 /**
@@ -158,6 +163,7 @@ export type LoadedProvider =
   | CacheProvider
   | SecretsProvider
   | S3Provider
+  | PluginRuntimeProvider
   | AgentProvider
   | WorkflowProvider;
 
@@ -218,6 +224,17 @@ const PROVIDER_RUNTIME_ENTRIES: Partial<
       router.service(S3Service, createS3Service(provider as S3Provider));
     },
   },
+  runtime: {
+    isProvider:
+      isPluginRuntimeProvider as (value: unknown) => value is LoadedProvider,
+    protoKind: ProtoProviderKind.RUNTIME,
+    registerService(router, provider) {
+      router.service(
+        PluginRuntimeProviderService,
+        createPluginRuntimeProviderService(provider as PluginRuntimeProvider),
+      );
+    },
+  },
   agent: {
     isProvider: isAgentProvider as (value: unknown) => value is LoadedProvider,
     protoKind: ProtoProviderKind.AGENT,
@@ -324,19 +341,11 @@ export async function runLoadedProvider(
   }
 
   const catalogPath = process.env[ENV_WRITE_CATALOG];
-  const manifestMetadataPath = process.env[ENV_WRITE_MANIFEST_METADATA];
-  if (catalogPath || manifestMetadataPath) {
+  if (catalogPath) {
     if (!isPluginProvider(provider)) {
-      throw new Error(
-        "static catalog and manifest metadata generation are only supported for plugin providers",
-      );
-    }
-    if (catalogPath) {
-      writeFileSync(catalogPath, catalogToYaml(provider.staticCatalog()), "utf8");
-    }
-    if (manifestMetadataPath && provider.supportsManifestMetadata()) {
-      provider.writeManifestMetadata(manifestMetadataPath);
+      throw new Error("static catalog generation is only supported for plugin providers");
     }
+    writeFileSync(catalogPath, catalogToYaml(provider.staticCatalog()), "utf8");
     return;
   }
 
@@ -364,9 +373,7 @@ export async function serve(provider: LoadedProvider): Promise<void> {
   if (!socketPath) {
     throw new Error(`${ENV_PROVIDER_SOCKET} is required`);
   }
-  if (existsSync(socketPath)) {
-    rmSync(socketPath);
-  }
+  rmSync(socketPath, { force: true });
 
   const handler = connectNodeAdapter({
     grpc: true,
@@ -393,9 +400,7 @@ export async function serve(provider: LoadedProvider): Promise<void> {
             server.close(() => resolveClose());
           });
         } finally {
-          if (existsSync(socketPath)) {
-            rmSync(socketPath);
-          }
+          rmSync(socketPath, { force: true });
         }
       }
     })();
@@ -482,6 +487,16 @@ export function createRuntimeService(
         });
       }
     },
+    async startProvider() {
+      try {
+        await provider.startProvider();
+      } catch (error) {
+        throw providerRuntimeError("start provider", error);
+      }
+      return create(StartRuntimeProviderResponseSchema, {
+        protocolVersion: CURRENT_PROTOCOL_VERSION,
+      });
+    },
   };
 }
 
@@ -548,6 +563,7 @@ export function createProviderService(
             request.connectionParams,
             request.context,
             request.invocationToken,
+            request.idempotencyKey,
           ),
         ),
       );
@@ -801,6 +817,7 @@ function providerRequest(
   connectionParams: Record<string, string>,
   requestContext?: ProtoRequestContext,
   invocationToken = "",
+  idempotencyKey = "",
 ): Request {
   const subject = requestContext?.subject;
   const credential = requestContext?.credential;
@@ -830,6 +847,7 @@ function providerRequest(
       ...(requestContext?.workflow ?? {}),
     },
     invocationToken,
+    idempotencyKey: idempotencyKey.trim(),
   };
 }
 
@@ -866,12 +884,12 @@ function providerHTTPSubjectResolutionContext(
 }
 
 function providerConnectedToken(
-  token?: ProtoIntegrationToken,
+  token?: ProtoPostConnectCredential,
 ): ConnectedToken {
   const metadataJson = token?.metadataJson ?? "";
   return {
     id: token?.id ?? "",
-    subjectId: token?.userId ?? "",
+    subjectId: token?.subjectId ?? "",
     integration: token?.integration ?? "",
     connection: token?.connection ?? "",
     instance: token?.instance ?? "",

package/src/s3.ts

@@ -20,6 +20,7 @@ import {
   PresignObjectResponseSchema,
   ReadObjectChunkSchema,
   S3 as S3Service,
+  S3ObjectAccess as S3ObjectAccessService,
   type ByteRange as ProtoByteRange,
   type ReadObjectRequest as ProtoReadObjectRequest,
   type S3ObjectMeta as ProtoS3ObjectMeta,
@@ -198,6 +199,16 @@ export interface PresignResult {
   headers: Record<string, string>;
 }
 
+/**
+ * Options used when creating a host-mediated object-access URL.
+ */
+export type ObjectAccessURLOptions = PresignOptions;
+
+/**
+ * Result returned by {@link S3.createObjectAccessURL} or {@link S3Object.createAccessURL}.
+ */
+export type ObjectAccessURL = PresignResult;
+
 /**
  * Accepted write body sources for the S3 client.
  */
@@ -509,7 +520,9 @@ export function createS3Service(
  * ```
  */
 export class S3 {
+  private readonly transport: ReturnType<typeof createGrpcTransport>;
   private readonly client: Client<typeof S3Service>;
+  private objectAccessClient?: Client<typeof S3ObjectAccessService>;
 
   constructor(name?: string) {
     const envName = s3SocketEnv(name);
@@ -538,6 +551,7 @@ export class S3 {
         : {}),
       interceptors,
     });
+    this.transport = transport;
     this.client = createClient(S3Service, transport);
   }
 
@@ -654,6 +668,62 @@ export class S3 {
     }
     return result;
   }
+
+  async createObjectAccessURL(
+    ref: ObjectRef,
+    options?: ObjectAccessURLOptions,
+  ): Promise<ObjectAccessURL> {
+    const requestedMethod = options?.method ?? PresignMethod.Get;
+    const response = await s3Rpc(() =>
+      this.s3ObjectAccessClient().createObjectAccessURL({
+        ref: toProtoObjectRef(ref),
+        method: toProtoPresignMethod(requestedMethod),
+        expiresSeconds: normalizeProtoInt(options?.expiresSeconds),
+        contentType: options?.contentType ?? "",
+        contentDisposition: options?.contentDisposition ?? "",
+        headers: cloneStringMap(options?.headers),
+      }),
+    );
+    const result: ObjectAccessURL = {
+      url: response.url,
+      method: response.method === ProtoPresignMethod.UNSPECIFIED
+        ? requestedMethod
+        : fromProtoPresignMethod(response.method),
+      headers: cloneStringMap(response.headers),
+    };
+    if (response.expiresAt) {
+      result.expiresAt = fromProtoTimestamp(response.expiresAt);
+    }
+    return result;
+  }
+
+  private s3ObjectAccessClient(): Client<typeof S3ObjectAccessService> {
+    if (!this.objectAccessClient) {
+      this.objectAccessClient = createClient(S3ObjectAccessService, this.transport);
+    }
+    return this.objectAccessClient;
+  }
+
+  async createObjectAccessUrl(
+    ref: ObjectRef,
+    options?: ObjectAccessURLOptions,
+  ): Promise<ObjectAccessURL> {
+    return await this.createObjectAccessURL(ref, options);
+  }
+
+  async createAccessURL(
+    ref: ObjectRef,
+    options?: ObjectAccessURLOptions,
+  ): Promise<ObjectAccessURL> {
+    return await this.createObjectAccessURL(ref, options);
+  }
+
+  async createAccessUrl(
+    ref: ObjectRef,
+    options?: ObjectAccessURLOptions,
+  ): Promise<ObjectAccessURL> {
+    return await this.createObjectAccessURL(ref, options);
+  }
 }
 
 /**
@@ -768,6 +838,17 @@ export class S3Object {
   async presign(options?: PresignOptions): Promise<PresignResult> {
     return await this.client.presignObject(this.ref, options);
   }
+
+  /**
+   * Creates a host-mediated object-access URL for the referenced object.
+   */
+  async createAccessURL(options?: ObjectAccessURLOptions): Promise<ObjectAccessURL> {
+    return await this.client.createObjectAccessURL(this.ref, options);
+  }
+
+  async createAccessUrl(options?: ObjectAccessURLOptions): Promise<ObjectAccessURL> {
+    return await this.createAccessURL(options);
+  }
 }
 
 function s3TransportOptions(rawTarget: string): {