every-plugin 0.0.5 → 0.1.0

package/dist/runtime/services/plugin.service.js

@@ -0,0 +1,338 @@
+import { Cache, Duration, Effect, Hash, Layer, Schedule } from "effect";
+import { PluginLoggerTag } from "../../plugin";
+import { PluginRuntimeError } from "../errors";
+import { validate } from "../validation";
+import { ModuleFederationService } from "./module-federation.service";
+import { SecretsService } from "./secrets.service";
+export class PluginService extends Effect.Tag("PluginService")() {
+	static Live = (registry, secrets) =>
+		Layer.effect(
+			PluginService,
+			Effect.gen(function* () {
+				const moduleFederationService = yield* ModuleFederationService;
+				const secretsService = yield* SecretsService;
+				const logger = yield* PluginLoggerTag;
+				const activePlugins = new Set();
+				const retrySchedule = Schedule.exponential(Duration.millis(100)).pipe(
+					Schedule.compose(Schedule.recurs(2)),
+				);
+				const resolveUrl = (baseUrl, version) =>
+					version && version !== "latest"
+						? baseUrl.replace("@latest", `@${version}`)
+						: baseUrl;
+				// Implementation functions
+				const loadPluginImpl = (pluginId) =>
+					Effect.gen(function* () {
+						const metadata = registry[pluginId];
+						if (!metadata) {
+							return yield* Effect.fail(
+								new PluginRuntimeError({
+									pluginId,
+									operation: "load-plugin",
+									cause: new Error(`Plugin ${pluginId} not found in registry`),
+									retryable: false,
+								}),
+							);
+						}
+						const url = resolveUrl(metadata.remoteUrl);
+						// Register and load the remote constructor
+						yield* moduleFederationService.registerRemote(pluginId, url).pipe(
+							Effect.mapError(
+								(error) =>
+									new PluginRuntimeError({
+										pluginId,
+										operation: "register-remote",
+										cause: error.cause,
+										retryable: true,
+									}),
+							),
+						);
+						logger.logInfo(`Trying to load ${pluginId}:${url}`);
+						const ctor = yield* moduleFederationService
+							.loadRemoteConstructor(pluginId, url)
+							.pipe(
+								Effect.mapError(
+									(error) =>
+										new PluginRuntimeError({
+											pluginId,
+											operation: "load-remote",
+											cause: error.cause,
+											retryable: false,
+										}),
+								),
+							);
+						return {
+							ctor,
+							metadata: {
+								pluginId,
+								version: metadata.version,
+								description: metadata.description,
+								type: metadata.type,
+							},
+						};
+					});
+				const instantiatePluginImpl = (pluginConstructor) =>
+					Effect.gen(function* () {
+						const instance = yield* Effect.try({
+							try: () => new pluginConstructor.ctor(),
+							catch: (error) =>
+								new PluginRuntimeError({
+									pluginId: pluginConstructor.metadata.pluginId,
+									operation: "instantiate-plugin",
+									cause:
+										error instanceof Error ? error : new Error(String(error)),
+									retryable: false,
+								}),
+						});
+						// Validate plugin ID matches
+						if (instance.id !== pluginConstructor.metadata.pluginId) {
+							return yield* Effect.fail(
+								new PluginRuntimeError({
+									pluginId: pluginConstructor.metadata.pluginId,
+									operation: "validate-plugin-id",
+									cause: new Error(
+										`Plugin ID mismatch: expected ${pluginConstructor.metadata.pluginId}, got ${instance.id}`,
+									),
+									retryable: false,
+								}),
+							);
+						}
+						return {
+							plugin: instance,
+							metadata: pluginConstructor.metadata,
+						};
+					});
+				const initializePluginImpl = (pluginInstance, config) =>
+					Effect.gen(function* () {
+						const { plugin } = pluginInstance;
+						// Validate raw config using plugin's Zod schema
+						const validatedConfig = yield* validate(
+							plugin.configSchema,
+							config,
+							plugin.id,
+							"config",
+						).pipe(
+							Effect.mapError(
+								(validationError) =>
+									new PluginRuntimeError({
+										pluginId: plugin.id,
+										operation: "validate-config",
+										cause: validationError.zodError,
+										retryable: false,
+									}),
+							),
+						);
+						// Hydrate secrets
+						const hydratedConfig = yield* secretsService.hydrateSecrets(
+							validatedConfig,
+							secrets,
+						);
+						// Validate hydrated config
+						const finalConfig = yield* validate(
+							plugin.configSchema,
+							hydratedConfig,
+							plugin.id,
+							"config",
+						).pipe(
+							Effect.mapError(
+								(validationError) =>
+									new PluginRuntimeError({
+										pluginId: plugin.id,
+										operation: "validate-hydrated-config",
+										cause: validationError.zodError,
+										retryable: false,
+									}),
+							),
+						);
+						// Initialize plugin with retry logic
+						const pluginLayer = Layer.succeed(PluginLoggerTag, logger);
+						yield* plugin.initialize(finalConfig).pipe(
+							Effect.mapError(
+								(error) =>
+									new PluginRuntimeError({
+										pluginId: plugin.id,
+										operation: "initialize-plugin",
+										cause: error,
+										retryable: error.retryable ?? false,
+									}),
+							),
+							Effect.retry(
+								retrySchedule.pipe(
+									Schedule.whileInput((error) => error.retryable),
+								),
+							),
+							Effect.provide(pluginLayer),
+						);
+						const initializedPlugin = {
+							plugin,
+							metadata: pluginInstance.metadata,
+							config: finalConfig, // TODO: config type
+						};
+						activePlugins.add(initializedPlugin);
+						return initializedPlugin;
+					});
+				// Cache functionality
+				const generateConfigHash = (config) => {
+					return Hash.structure(config).toString();
+				};
+				// Store original config by hash for cache lookup
+				const configByHash = new Map();
+				const cache = yield* Cache.make({
+					capacity: 1024,
+					timeToLive: Duration.minutes(60),
+					lookup: (cacheKey) => {
+						const [pluginId, configHash] = cacheKey.split(":");
+						if (!pluginId || !configHash) {
+							return Effect.fail(
+								new PluginRuntimeError({
+									operation: "cache-lookup",
+									cause: new Error(`Invalid cache key format: ${cacheKey}`),
+									retryable: false,
+								}),
+							);
+						}
+						const config = configByHash.get(configHash);
+						if (!config) {
+							return Effect.fail(
+								new PluginRuntimeError({
+									operation: "cache-lookup",
+									cause: new Error(`No config found for hash: ${configHash}`),
+									retryable: false,
+								}),
+							);
+						}
+						return Effect.gen(function* () {
+							const ctor = yield* loadPluginImpl(pluginId);
+							const instance = yield* instantiatePluginImpl(ctor);
+							return yield* initializePluginImpl(instance, config);
+						});
+					},
+				});
+				return {
+					loadPlugin: loadPluginImpl,
+					instantiatePlugin: instantiatePluginImpl,
+					initializePlugin: initializePluginImpl,
+					executePlugin: (initializedPlugin, input) =>
+						Effect.gen(function* () {
+							const { plugin } = initializedPlugin;
+							// Validate input
+							const validatedInput = yield* validate(
+								plugin.inputSchema,
+								input,
+								plugin.id,
+								"input",
+							).pipe(
+								Effect.mapError((validationError) => {
+									console.log("got input error", validationError.zodError);
+									return new PluginRuntimeError({
+										pluginId: plugin.id,
+										operation: "validate-input",
+										cause: validationError.zodError,
+										retryable: false,
+									});
+								}),
+							);
+							// Execute plugin
+							const pluginLayer = Layer.succeed(PluginLoggerTag, logger);
+							const output = yield* plugin.execute(validatedInput).pipe(
+								Effect.mapError(
+									(error) =>
+										new PluginRuntimeError({
+											pluginId: plugin.id,
+											operation: "execute-plugin",
+											cause: error,
+											retryable: error.retryable ?? false,
+										}),
+								),
+								Effect.retry(
+									retrySchedule.pipe(
+										Schedule.whileInput((error) => error.retryable),
+									),
+								),
+								Effect.provide(pluginLayer),
+							);
+							// Validate output
+							const validatedOutput = yield* validate(
+								plugin.outputSchema,
+								output,
+								plugin.id,
+								"output",
+							).pipe(
+								Effect.mapError((validationError) => {
+									console.log("got output error", validationError.zodError);
+									return new PluginRuntimeError({
+										pluginId: plugin.id,
+										operation: "validate-output",
+										cause: validationError.zodError,
+										retryable: false,
+									});
+								}),
+							);
+							return validatedOutput;
+						}),
+					usePlugin: (pluginId, config) => {
+						const configHash = generateConfigHash(config);
+						const cacheKey = `${pluginId}:${configHash}`;
+						// Store config for cache lookup
+						configByHash.set(configHash, config);
+						return cache.get(cacheKey).pipe(Effect.map((plugin) => plugin));
+					},
+					shutdownPlugin: (plugin) =>
+						Effect.gen(function* () {
+							const pluginLayer = Layer.succeed(PluginLoggerTag, logger);
+							// Remove from active plugins
+							activePlugins.delete(plugin);
+							// Try to invalidate from cache if it exists there
+							const configHash = generateConfigHash(plugin.config);
+							const cacheKey = `${plugin.plugin.id}:${configHash}`;
+							yield* cache.invalidate(cacheKey).pipe(
+								Effect.catchAll(() => Effect.void), // Ignore if not in cache
+							);
+							// Clean up config hash
+							configByHash.delete(configHash);
+							// Shutdown the plugin
+							yield* plugin.plugin.shutdown().pipe(
+								Effect.mapError(
+									(error) =>
+										new PluginRuntimeError({
+											pluginId: plugin.plugin.id,
+											operation: "shutdown-plugin",
+											cause: error,
+											retryable: false,
+										}),
+								),
+								Effect.provide(pluginLayer),
+							);
+						}),
+					clearCache: () =>
+						Effect.sync(() => {
+							configByHash.clear();
+						}),
+					cleanup: () =>
+						Effect.gen(function* () {
+							const pluginLayer = Layer.succeed(PluginLoggerTag, logger);
+							// Call shutdown on all active plugins
+							for (const initializedPlugin of activePlugins) {
+								yield* initializedPlugin.plugin.shutdown().pipe(
+									Effect.catchAll((error) => {
+										// Log shutdown errors but don't fail the cleanup
+										return logger.logError(
+											`Failed to shutdown plugin ${initializedPlugin.plugin.id}`,
+											error,
+											{ pluginId: initializedPlugin.plugin.id },
+										);
+									}),
+									Effect.provide(pluginLayer),
+								);
+							}
+							// Clear the active plugins set
+							activePlugins.clear();
+							// Clear cache
+							configByHash.clear();
+						}),
+				};
+			}),
+		);
+}
+export const PluginServiceTag = PluginService;
+export const PluginServiceLive = PluginService.Live;

package/dist/runtime/services/secrets.service.d.ts

@@ -0,0 +1,37 @@
+import { Effect, Layer } from "effect";
+import { PluginRuntimeError } from "../errors";
+import type { SecretsConfig } from "../types";
+export interface ISecretsService {
+	readonly hydrateSecrets: (
+		config: any, // TODO:
+		secretsConfig: SecretsConfig,
+	) => Effect.Effect<any, PluginRuntimeError>;
+}
+declare const SecretsService_base: import("effect/Context").TagClass<
+	SecretsService,
+	"SecretsService",
+	ISecretsService
+> &
+	Effect.Tag.Proxy<SecretsService, ISecretsService> & {
+		use: <X>(
+			body: (_: ISecretsService) => X,
+		) => [X] extends [Effect.Effect<infer A, infer E, infer R>]
+			? Effect.Effect<A, E, R | SecretsService>
+			: [X] extends [PromiseLike<infer A_1>]
+				? Effect.Effect<
+						A_1,
+						import("effect/Cause").UnknownException,
+						SecretsService
+					>
+				: Effect.Effect<X, never, SecretsService>;
+	};
+export declare class SecretsService extends SecretsService_base {
+	static Live: (
+		secrets: SecretsConfig,
+	) => Layer.Layer<SecretsService, never, never>;
+}
+export declare const SecretsServiceTag: typeof SecretsService;
+export declare const SecretsServiceLive: (
+	secrets: SecretsConfig,
+) => Layer.Layer<SecretsService, never, never>;
+//# sourceMappingURL=secrets.service.d.ts.map

package/dist/runtime/services/secrets.service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"secrets.service.d.ts","sourceRoot":"","sources":["../../../src/runtime/services/secrets.service.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,QAAQ,CAAC;AACvC,OAAO,EAAE,kBAAkB,EAAE,MAAM,WAAW,CAAC;AAC/C,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAE9C,MAAM,WAAW,eAAe;IAC/B,QAAQ,CAAC,cAAc,EAAE,CACxB,MAAM,EAAE,GAAG,EAAE,QAAQ;IACrB,aAAa,EAAE,aAAa,KACxB,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,kBAAkB,CAAC,CAAC;CAC5C;;;;AAED,qBAAa,cAAe,SAAQ,mBAGjC;IACF,MAAM,CAAC,IAAI,GAAI,SAAS,aAAa,+CAiCjC;CACJ;AAED,eAAO,MAAM,iBAAiB,uBAAiB,CAAC;AAChD,eAAO,MAAM,kBAAkB,YArCN,aAAa,8CAqCe,CAAC"}

package/dist/runtime/services/secrets.service.js

@@ -0,0 +1,36 @@
+import { Effect, Layer } from "effect";
+import { PluginRuntimeError } from "../errors";
+export class SecretsService extends Effect.Tag("SecretsService")() {
+	static Live = (secrets) =>
+		Layer.succeed(SecretsService, {
+			hydrateSecrets: (config, secretsConfig) =>
+				Effect.gen(function* () {
+					if (!config || typeof config !== "object" || !config.secrets) {
+						return config;
+					}
+					try {
+						const configString = JSON.stringify(config);
+						let hydratedString = configString;
+						// Use provided secrets or fallback to secretsConfig
+						const effectiveSecrets = { ...secretsConfig, ...secrets };
+						// Simple template replacement for {{SECRET_NAME}} patterns
+						for (const [key, value] of Object.entries(effectiveSecrets)) {
+							const pattern = new RegExp(`{{${key}}}`, "g");
+							hydratedString = hydratedString.replace(pattern, value);
+						}
+						return JSON.parse(hydratedString);
+					} catch (error) {
+						return yield* Effect.fail(
+							new PluginRuntimeError({
+								operation: "hydrate-secrets",
+								cause:
+									error instanceof Error ? error : new Error(String(error)),
+								retryable: false,
+							}),
+						);
+					}
+				}),
+		});
+}
+export const SecretsServiceTag = SecretsService;
+export const SecretsServiceLive = SecretsService.Live;

package/dist/runtime/services.d.ts

@@ -0,0 +1,4 @@
+export * from "./services/module-federation.service";
+export * from "./services/plugin.service";
+export * from "./services/secrets.service";
+//# sourceMappingURL=services.d.ts.map

package/dist/runtime/services.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"services.d.ts","sourceRoot":"","sources":["../../src/runtime/services.ts"],"names":[],"mappings":"AAAA,cAAc,sCAAsC,CAAC;AACrD,cAAc,2BAA2B,CAAC;AAC1C,cAAc,4BAA4B,CAAC"}

package/dist/runtime/services.js

@@ -0,0 +1,3 @@
+export * from "./services/module-federation.service";
+export * from "./services/plugin.service";
+export * from "./services/secrets.service";

package/dist/runtime/streaming.d.ts

@@ -0,0 +1,29 @@
+import { Effect, Stream } from "effect";
+import type { SourcePlugin } from "../source";
+import { PluginRuntimeError } from "./errors";
+import type { InitializedPlugin } from "./types";
+import z from "zod";
+export interface SourceStreamOptions<TItem = unknown> {
+	maxItems?: number;
+	maxInvocations?: number;
+	stopWhenEmpty?: boolean;
+}
+/**
+ * Create a stream from a source plugin
+ * Note: Validation is handled at the runtime level before this function is called
+ */
+export declare const createSourceStream: <
+	T extends SourcePlugin,
+	TInput extends z.infer<T["inputSchema"]>,
+	TItem,
+>(
+	initializedPlugin: InitializedPlugin<T>,
+	executePlugin: (
+		plugin: InitializedPlugin<T>,
+		input: TInput,
+	) => Effect.Effect<any, PluginRuntimeError>,
+	input: TInput,
+	options: SourceStreamOptions<TItem> | undefined,
+	isStreamable: boolean,
+) => Stream.Stream<TItem, PluginRuntimeError>;
+//# sourceMappingURL=streaming.d.ts.map

package/dist/runtime/streaming.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"streaming.d.ts","sourceRoot":"","sources":["../../src/runtime/streaming.ts"],"names":[],"mappings":"AAAA,OAAO,EAAY,MAAM,EAAU,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC1D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAC9C,OAAO,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAC9C,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AAEjD,OAAO,CAAC,MAAM,KAAK,CAAC;AAGpB,MAAM,WAAW,mBAAmB,CAAC,KAAK,GAAG,OAAO;IACnD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,aAAa,CAAC,EAAE,OAAO,CAAC;CACxB;AAMD;;;GAGG;AACH,eAAO,MAAM,kBAAkB,GAC9B,CAAC,SAAS,YAAY,EACtB,MAAM,SAAS,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,EACxC,KAAK,EAEL,mBAAmB,iBAAiB,CAAC,CAAC,CAAC,EACvC,eAAe,CAAC,MAAM,EAAE,iBAAiB,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,KAAK,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,kBAAkB,CAAC,EACtG,OAAO,MAAM,EACb,SAAS,mBAAmB,CAAC,KAAK,CAAC,YAAK,EACxC,cAAc,OAAO,KACnB,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,kBAAkB,CAqEzC,CAAC"}

package/dist/runtime/streaming.js

@@ -0,0 +1,59 @@
+import { Duration, Effect, Option, Stream } from "effect";
+const extractDelayFromState = (state) => {
+    return state?.nextPollMs ? Duration.millis(state.nextPollMs) : null;
+};
+/**
+ * Create a stream from a source plugin
+ * Note: Validation is handled at the runtime level before this function is called
+ */
+export const createSourceStream = (initializedPlugin, executePlugin, input, options = {}, isStreamable) => {
+    // For non-streamable procedures, execute once and return items
+    if (!isStreamable) {
+        return Stream.fromEffect(executePlugin(initializedPlugin, input).pipe(Effect.map((result) => {
+            const resultObj = result;
+            const items = Array.isArray(resultObj.items)
+                ? resultObj.items
+                : resultObj.item
+                    ? [resultObj.item]
+                    : [];
+            return items;
+        }))).pipe(Stream.flatMap(items => Stream.fromIterable(items)));
+    }
+    const initialState = input.state;
+    const stream = Stream.unfoldEffect({ state: initialState, invocations: 0 }, ({ state: currentState, invocations }) => Effect.gen(function* () {
+        console.log(`[STREAM] Invocation ${invocations}:`);
+        console.log(`[STREAM] - Current state:`, JSON.stringify(currentState, null, 2));
+        console.log(`[STREAM] - Input:`, JSON.stringify(input, null, 2));
+        // Check invocation limit if specified
+        if (options.maxInvocations && invocations >= options.maxInvocations) {
+            console.log(`[STREAM] - Stopping: maxInvocations (${options.maxInvocations}) reached`);
+            return Option.none();
+        }
+        // Extract timing from plugin state (nextPollMs)
+        const nextDelay = extractDelayFromState(currentState) || Duration.minutes(1); // 1min default
+        yield* Effect.sleep(nextDelay);
+        // Execute plugin
+        const pluginInput = { ...input, state: currentState };
+        console.log(`[STREAM] - Plugin input:`, JSON.stringify(pluginInput, null, 2));
+        const result = yield* executePlugin(initializedPlugin, pluginInput);
+        const resultObj = result;
+        const items = Array.isArray(resultObj.items)
+            ? resultObj.items
+            : [];
+        const nextState = resultObj.nextState;
+        console.log(`[STREAM] - Plugin output:`, JSON.stringify(resultObj, null, 2));
+        console.log(`[STREAM] - Items count: ${items.length}`);
+        console.log(`[STREAM] - Next state:`, JSON.stringify(nextState, null, 2));
+        // Check stop conditions ONLY
+        if (options.stopWhenEmpty && items.length === 0) {
+            console.log(`[STREAM] - Stopping: stopWhenEmpty=true and items.length=0`);
+            return Option.none();
+        }
+        console.log(`[STREAM] - Continuing to next iteration`);
+        return Option.some([items, { state: nextState, invocations: invocations + 1 }]);
+    })).pipe(
+    // Flatten items from each iteration
+    Stream.flatMap(items => Stream.fromIterable(items)));
+    // Apply maxItems limit if specified
+    return options.maxItems ? stream.pipe(Stream.take(options.maxItems)) : stream;
+};

package/dist/runtime/streaming.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=streaming.test.d.ts.map

package/dist/runtime/streaming.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"streaming.test.d.ts","sourceRoot":"","sources":["../../src/runtime/streaming.test.ts"],"names":[],"mappings":""}