every-plugin 0.0.6 → 0.1.0

package/dist/runtime/streaming.js

@@ -0,0 +1,59 @@
+import { Duration, Effect, Option, Stream } from "effect";
+const extractDelayFromState = (state) => {
+    return state?.nextPollMs ? Duration.millis(state.nextPollMs) : null;
+};
+/**
+ * Create a stream from a source plugin
+ * Note: Validation is handled at the runtime level before this function is called
+ */
+export const createSourceStream = (initializedPlugin, executePlugin, input, options = {}, isStreamable) => {
+    // For non-streamable procedures, execute once and return items
+    if (!isStreamable) {
+        return Stream.fromEffect(executePlugin(initializedPlugin, input).pipe(Effect.map((result) => {
+            const resultObj = result;
+            const items = Array.isArray(resultObj.items)
+                ? resultObj.items
+                : resultObj.item
+                    ? [resultObj.item]
+                    : [];
+            return items;
+        }))).pipe(Stream.flatMap(items => Stream.fromIterable(items)));
+    }
+    const initialState = input.state;
+    const stream = Stream.unfoldEffect({ state: initialState, invocations: 0 }, ({ state: currentState, invocations }) => Effect.gen(function* () {
+        console.log(`[STREAM] Invocation ${invocations}:`);
+        console.log(`[STREAM] - Current state:`, JSON.stringify(currentState, null, 2));
+        console.log(`[STREAM] - Input:`, JSON.stringify(input, null, 2));
+        // Check invocation limit if specified
+        if (options.maxInvocations && invocations >= options.maxInvocations) {
+            console.log(`[STREAM] - Stopping: maxInvocations (${options.maxInvocations}) reached`);
+            return Option.none();
+        }
+        // Extract timing from plugin state (nextPollMs)
+        const nextDelay = extractDelayFromState(currentState) || Duration.minutes(1); // 1min default
+        yield* Effect.sleep(nextDelay);
+        // Execute plugin
+        const pluginInput = { ...input, state: currentState };
+        console.log(`[STREAM] - Plugin input:`, JSON.stringify(pluginInput, null, 2));
+        const result = yield* executePlugin(initializedPlugin, pluginInput);
+        const resultObj = result;
+        const items = Array.isArray(resultObj.items)
+            ? resultObj.items
+            : [];
+        const nextState = resultObj.nextState;
+        console.log(`[STREAM] - Plugin output:`, JSON.stringify(resultObj, null, 2));
+        console.log(`[STREAM] - Items count: ${items.length}`);
+        console.log(`[STREAM] - Next state:`, JSON.stringify(nextState, null, 2));
+        // Check stop conditions ONLY
+        if (options.stopWhenEmpty && items.length === 0) {
+            console.log(`[STREAM] - Stopping: stopWhenEmpty=true and items.length=0`);
+            return Option.none();
+        }
+        console.log(`[STREAM] - Continuing to next iteration`);
+        return Option.some([items, { state: nextState, invocations: invocations + 1 }]);
+    })).pipe(
+    // Flatten items from each iteration
+    Stream.flatMap(items => Stream.fromIterable(items)));
+    // Apply maxItems limit if specified
+    return options.maxItems ? stream.pipe(Stream.take(options.maxItems)) : stream;
+};

package/dist/runtime/streaming.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=streaming.test.d.ts.map

package/dist/runtime/streaming.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"streaming.test.d.ts","sourceRoot":"","sources":["../../src/runtime/streaming.test.ts"],"names":[],"mappings":""}

package/dist/runtime/streaming.test.js

@@ -0,0 +1,271 @@
+import { Effect, Stream } from "effect";
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import { createPluginRuntime, PluginRuntime } from "./index";
+// Test registry with our example source plugin
+const TEST_REGISTRY = {
+    "test-source-plugin": {
+        remoteUrl: "http://localhost:3001/remoteEntry.js",
+        type: "source",
+        version: "0.0.1",
+        description: "Example source plugin for streaming tests",
+    },
+    "test-plugin": {
+        remoteUrl: "http://localhost:3000/remoteEntry.js",
+        type: "transformer",
+        version: "0.0.1",
+        description: "Local template plugin for testing",
+    },
+};
+const TEST_CONFIG = {
+    variables: {
+        baseUrl: "http://localhost:1337",
+        timeout: 5000,
+    },
+    secrets: {
+        apiKey: "test-api-key-value",
+    },
+};
+// Contract-based input for search procedure
+const TEST_SEARCH_INPUT = {
+    procedure: "search",
+    input: {
+        query: "test query",
+        limit: 10,
+    },
+    state: null,
+};
+// Contract-based input for getById procedure
+const TEST_GETBYID_INPUT = {
+    procedure: "getById",
+    input: {
+        id: "test-id-123",
+    },
+    state: null,
+};
+// Contract-based input for getBulk procedure
+const TEST_GETBULK_INPUT = {
+    procedure: "getBulk",
+    input: {
+        ids: ["id1", "id2", "id3"],
+    },
+    state: null,
+};
+const SECRETS_CONFIG = {
+    API_KEY: "test-api-key-value",
+};
+describe("Plugin Streaming", () => {
+    let runtime;
+    beforeEach(() => {
+        runtime = createPluginRuntime({
+            registry: TEST_REGISTRY,
+            secrets: SECRETS_CONFIG,
+        });
+    });
+    afterEach(async () => {
+        if (runtime) {
+            await runtime.dispose();
+        }
+    });
+    it("should respect maxItems limit", async () => {
+        const result = await runtime.runPromise(Effect.gen(function* () {
+            const pluginRuntime = yield* PluginRuntime;
+            // Start from Phase 3 state that immediately produces items
+            const inputWithProductiveState = {
+                ...TEST_SEARCH_INPUT,
+                state: { phase: "historical", status: "processing", nextPollMs: 10 }
+            };
+            const stream = yield* pluginRuntime.streamPlugin("test-source-plugin", TEST_CONFIG, inputWithProductiveState, {
+                maxItems: 3, // Strict limit
+                maxInvocations: 3
+            });
+            const items = yield* stream.pipe(Stream.runCollect);
+            return Array.from(items);
+        }));
+        expect(result.length).toBeLessThanOrEqual(3);
+        expect(result.length).toBeGreaterThan(0);
+    }, 4000);
+    it("should validate invalid state and fail with proper error", async () => {
+        const result = await runtime.runPromise(Effect.gen(function* () {
+            const pluginRuntime = yield* PluginRuntime;
+            // Use invalid state that doesn't match plugin's stateSchema
+            const inputWithInvalidState = {
+                ...TEST_SEARCH_INPUT,
+                state: {
+                    pageNumber: 10, // Invalid - plugin expects { phase, status?, jobId?, lastId?, nextPollMs? }
+                    invalidField: "should not be here",
+                },
+            };
+            // The error should occur when creating the stream, not when running it
+            return yield* pluginRuntime.streamPlugin("test-source-plugin", TEST_CONFIG, inputWithInvalidState, {
+                stopWhenEmpty: true,
+            }).pipe(Effect.flatMap(() => 
+            // If we get here, validation didn't work
+            Effect.succeed("validation-should-have-failed")), Effect.catchTag("PluginRuntimeError", (error) => {
+                // Should catch state validation error
+                expect(error.operation).toBe("validate-state");
+                expect(error.retryable).toBe(false);
+                expect(error.pluginId).toBe("test-source-plugin");
+                expect(error.cause).toBeDefined();
+                console.debug("Caught expected state validation error:", {
+                    operation: error.operation,
+                    pluginId: error.pluginId,
+                    retryable: error.retryable,
+                    cause: error.cause?.message,
+                });
+                return Effect.succeed("state-validation-error-handled");
+            }), Effect.catchAll((unexpectedError) => {
+                console.error("Unexpected error type:", unexpectedError);
+                expect.fail(`Expected PluginRuntimeError but got: ${unexpectedError._tag || typeof unexpectedError}`);
+                return Effect.succeed("should-not-reach-here");
+            }));
+        }));
+        expect(result).toBe("state-validation-error-handled");
+    }, 4000);
+    it("should handle stopWhenEmpty option", async () => {
+        const result = await runtime.runPromise(Effect.gen(function* () {
+            const pluginRuntime = yield* PluginRuntime;
+            // Use the deterministic "empty" phase that always returns 0 items
+            const inputWithEmptyState = {
+                ...TEST_SEARCH_INPUT,
+                state: {
+                    phase: "empty",
+                    nextPollMs: 10,
+                },
+            };
+            const stream = yield* pluginRuntime.streamPlugin("test-source-plugin", TEST_CONFIG, inputWithEmptyState, {
+                stopWhenEmpty: true,
+                maxInvocations: 3, // Allow a few iterations to verify it stops
+            });
+            const items = yield* stream.pipe(Stream.runCollect);
+            return Array.from(items);
+        }));
+        // Should stop immediately when no items are returned from empty phase
+        expect(Array.isArray(result)).toBe(true);
+        expect(result.length).toBe(0); // Empty phase always returns 0 items
+    }, 4000);
+    it("should validate plugin type", async () => {
+        const result = await runtime.runPromise(Effect.gen(function* () {
+            const pluginRuntime = yield* PluginRuntime;
+            return yield* pluginRuntime.streamPlugin("test-plugin", TEST_CONFIG, TEST_SEARCH_INPUT).pipe(Effect.catchTag("PluginRuntimeError", (error) => {
+                expect(error.operation).toBe("stream-plugin-validate");
+                expect(error.cause?.message).toContain("not a source plugin");
+                return Effect.succeed("validation-error-handled");
+            }));
+        }));
+        expect(result).toBe("validation-error-handled");
+    });
+    it("should handle workflow phases correctly", async () => {
+        const pluginRuntime = await runtime.runPromise(PluginRuntime);
+        const plugin = await runtime.runPromise(pluginRuntime.usePlugin("test-source-plugin", TEST_CONFIG));
+        // Phase 1: null state -> historical job (empty items)
+        const phase1 = await runtime.runPromise(pluginRuntime.executePlugin(plugin, TEST_SEARCH_INPUT));
+        const phase1Result = phase1;
+        expect(phase1Result.items).toHaveLength(0);
+        expect(phase1Result.nextState.phase).toBe("historical");
+        expect(phase1Result.nextState.jobId).toMatch(/^hist_\d+$/);
+        expect(phase1Result.nextState.nextPollMs).toBe(10);
+        // Phase 2: historical job -> processing (empty items)
+        const phase2Input = { ...TEST_SEARCH_INPUT, state: phase1Result.nextState };
+        const phase2 = await runtime.runPromise(pluginRuntime.executePlugin(plugin, phase2Input));
+        const phase2Result = phase2;
+        expect(phase2Result.items).toHaveLength(0);
+        expect(phase2Result.nextState.phase).toBe("historical");
+        expect(phase2Result.nextState.status).toBe("processing");
+        expect(phase2Result.nextState.nextPollMs).toBe(10);
+        // Phase 3: processing -> historical items + realtime transition
+        const phase3Input = { ...TEST_SEARCH_INPUT, state: phase2Result.nextState };
+        const phase3 = await runtime.runPromise(pluginRuntime.executePlugin(plugin, phase3Input));
+        const phase3Result = phase3;
+        expect(phase3Result.items.length).toBeGreaterThan(0);
+        expect(phase3Result.items[0]?.content).toContain("Historical");
+        expect(phase3Result.nextState.phase).toBe("realtime");
+        expect(phase3Result.nextState.lastId).toMatch(/^hist_end_\d+$/);
+        expect(phase3Result.nextState.nextPollMs).toBe(10);
+        // Phase 4: realtime polling (may have items)
+        const phase4Input = { ...TEST_SEARCH_INPUT, state: phase3Result.nextState };
+        const phase4 = await runtime.runPromise(pluginRuntime.executePlugin(plugin, phase4Input));
+        const phase4Result = phase4;
+        expect(phase4Result.nextState.phase).toBe("realtime");
+        // Items may be empty or contain real-time items (random)
+        if (phase4Result.items.length > 0) {
+            expect(phase4Result.items[0]?.content).toContain("Real-time");
+        }
+    }, 4000);
+    describe("Contract-based Procedures", () => {
+        it("should execute getById procedure", async () => {
+            const result = await runtime.runPromise(Effect.gen(function* () {
+                const pluginRuntime = yield* PluginRuntime;
+                const plugin = yield* pluginRuntime.usePlugin("test-source-plugin", TEST_CONFIG);
+                const output = yield* pluginRuntime.executePlugin(plugin, TEST_GETBYID_INPUT);
+                return output;
+            }));
+            expect(result).toBeDefined();
+            const typedResult = result;
+            expect(typedResult.item).toBeDefined();
+            expect(typedResult.item.externalId).toBe("test-id-123");
+            expect(typedResult.item.content).toContain("Content for item test-id-123");
+        });
+        it("should execute getBulk procedure", async () => {
+            const result = await runtime.runPromise(Effect.gen(function* () {
+                const pluginRuntime = yield* PluginRuntime;
+                const plugin = yield* pluginRuntime.usePlugin("test-source-plugin", TEST_CONFIG);
+                const output = yield* pluginRuntime.executePlugin(plugin, TEST_GETBULK_INPUT);
+                return output;
+            }));
+            expect(result).toBeDefined();
+            const typedResult = result;
+            expect(typedResult.items).toBeDefined();
+            expect(Array.isArray(typedResult.items)).toBe(true);
+            expect(typedResult.items.length).toBe(3);
+            expect(typedResult.items[0].externalId).toBe("id1");
+            expect(typedResult.items[1].externalId).toBe("id2");
+            expect(typedResult.items[2].externalId).toBe("id3");
+        });
+        it("should execute search procedure", async () => {
+            const result = await runtime.runPromise(Effect.gen(function* () {
+                const pluginRuntime = yield* PluginRuntime;
+                const plugin = yield* pluginRuntime.usePlugin("test-source-plugin", TEST_CONFIG);
+                // Use Phase 3 state to get historical items
+                const searchInputWithState = {
+                    ...TEST_SEARCH_INPUT,
+                    state: { phase: "historical", status: "processing" }
+                };
+                const output = yield* pluginRuntime.executePlugin(plugin, searchInputWithState);
+                return output;
+            }));
+            expect(result).toBeDefined();
+            const typedResult = result;
+            expect(typedResult.items).toBeDefined();
+            expect(Array.isArray(typedResult.items)).toBe(true);
+            expect(typedResult.items.length).toBeGreaterThan(0);
+            expect(typedResult.items[0].content).toContain("test query");
+            expect(typedResult.nextState).toBeDefined();
+        });
+        it("should stream getById procedure (single item)", async () => {
+            const result = await runtime.runPromise(Effect.gen(function* () {
+                const pluginRuntime = yield* PluginRuntime;
+                const stream = yield* pluginRuntime.streamPlugin("test-source-plugin", TEST_CONFIG, TEST_GETBYID_INPUT, {
+                    maxItems: 5, // Safety limit
+                });
+                const items = yield* stream.pipe(Stream.runCollect);
+                return Array.from(items);
+            }));
+            expect(result).toBeDefined();
+            expect(result.length).toBe(1);
+            expect(result[0].externalId).toBe("test-id-123");
+        });
+        it("should stream getBulk procedure", async () => {
+            const result = await runtime.runPromise(Effect.gen(function* () {
+                const pluginRuntime = yield* PluginRuntime;
+                const stream = yield* pluginRuntime.streamPlugin("test-source-plugin", TEST_CONFIG, TEST_GETBULK_INPUT, {
+                    maxItems: 5, // Safety limit
+                });
+                const items = yield* stream.pipe(Stream.runCollect);
+                return Array.from(items);
+            }));
+            expect(result).toBeDefined();
+            expect(result.length).toBe(3);
+            expect(result.map(item => item.externalId)).toEqual(["id1", "id2", "id3"]);
+        });
+    });
+});

package/dist/runtime/test-utils.d.ts

@@ -0,0 +1,3 @@
+import type { PluginLogger } from "../plugin";
+export declare const createTestLogger: () => PluginLogger;
+//# sourceMappingURL=test-utils.d.ts.map

package/dist/runtime/test-utils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"test-utils.d.ts","sourceRoot":"","sources":["../../src/runtime/test-utils.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAE9C,eAAO,MAAM,gBAAgB,QAAO,YAYlC,CAAC"}

package/dist/runtime/test-utils.js

@@ -0,0 +1,13 @@
+import { Effect } from "effect";
+export const createTestLogger = () => ({
+	logInfo: (message, context) =>
+		Effect.sync(() => console.log(`[INFO] ${message}`, context || "")),
+	logWarning: (message, context) =>
+		Effect.sync(() => console.warn(`[WARNING] ${message}`, context || "")),
+	logError: (message, error, context) =>
+		Effect.sync(() =>
+			console.error(`[ERROR] ${message}`, error, context || ""),
+		),
+	logDebug: (message, context) =>
+		Effect.sync(() => console.log(`[DEBUG] ${message}`, context || "")),
+});

package/dist/runtime/types.d.ts

@@ -1,40 +1,40 @@
 import type { z } from "zod";
 import type { Plugin, PluginLogger, PluginRegistry } from "../plugin";
 export interface PluginRuntimeConfig {
-    registry: PluginRegistry;
-    secrets?: SecretsConfig;
-    logger?: PluginLogger;
+	registry: PluginRegistry;
+	secrets?: SecretsConfig;
+	logger?: PluginLogger;
 }
 export interface SecretsConfig {
-    [key: string]: string;
+	[key: string]: string;
 }
 export type AnyPlugin = Plugin<z.ZodTypeAny, z.ZodTypeAny, z.ZodTypeAny>;
 export interface PluginConstructor {
-    readonly ctor: new () => AnyPlugin;
-    readonly metadata: {
-        readonly pluginId: string;
-        readonly version?: string;
-        readonly description?: string;
-        readonly type?: string;
-    };
+	readonly ctor: new () => AnyPlugin;
+	readonly metadata: {
+		readonly pluginId: string;
+		readonly version?: string;
+		readonly description?: string;
+		readonly type?: string;
+	};
 }
 export interface PluginInstance<T extends AnyPlugin = AnyPlugin> {
-    readonly plugin: T;
-    readonly metadata: {
-        readonly pluginId: string;
-        readonly version?: string;
-        readonly description?: string;
-        readonly type?: string;
-    };
+	readonly plugin: T;
+	readonly metadata: {
+		readonly pluginId: string;
+		readonly version?: string;
+		readonly description?: string;
+		readonly type?: string;
+	};
 }
 export interface InitializedPlugin<T extends AnyPlugin = AnyPlugin> {
-    readonly plugin: T;
-    readonly metadata: {
-        readonly pluginId: string;
-        readonly version?: string;
-        readonly description?: string;
-        readonly type?: string;
-    };
-    readonly config: z.infer<T["configSchema"]>;
+	readonly plugin: T;
+	readonly metadata: {
+		readonly pluginId: string;
+		readonly version?: string;
+		readonly description?: string;
+		readonly type?: string;
+	};
+	readonly config: z.infer<T["configSchema"]>;
 }
-//# sourceMappingURL=types.d.ts.map
+//# sourceMappingURL=types.d.ts.map

package/dist/runtime/validation.d.ts

@@ -5,6 +5,6 @@ export declare const validate: <T>(
 	schema: z.ZodSchema<T>,
 	data: unknown,
 	pluginId: string,
-	stage: "config" | "input" | "output",
+	stage: "config" | "input" | "output" | "state",
 ) => Effect.Effect<T, ValidationError>;
 //# sourceMappingURL=validation.d.ts.map

package/dist/runtime/validation.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"validation.d.ts","sourceRoot":"","sources":["../../src/runtime/validation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAChC,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAC7B,OAAO,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAE3C,eAAO,MAAM,QAAQ,GAAI,CAAC,EACzB,QAAQ,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EACtB,MAAM,OAAO,EACb,UAAU,MAAM,EAChB,OAAO,QAAQ,GAAG,OAAO,GAAG,QAAQ,KAClC,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,eAAe,CAahC,CAAC"}
+{"version":3,"file":"validation.d.ts","sourceRoot":"","sources":["../../src/runtime/validation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAChC,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAC7B,OAAO,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAE3C,eAAO,MAAM,QAAQ,GAAI,CAAC,EACzB,QAAQ,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EACtB,MAAM,OAAO,EACb,UAAU,MAAM,EAChB,OAAO,QAAQ,GAAG,OAAO,GAAG,QAAQ,GAAG,OAAO,KAC5C,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,eAAe,CAahC,CAAC"}

package/dist/runtime/validation.js

@@ -1,13 +1,16 @@
 import { Effect } from "effect";
 import { ValidationError } from "./errors";
-export const validate = (schema, data, pluginId, stage) => Effect.gen(function* () {
-    const result = schema.safeParse(data);
-    if (result.success) {
-        return result.data;
-    }
-    return yield* Effect.fail(new ValidationError({
-        pluginId,
-        stage,
-        zodError: result.error,
-    }));
-});
+export const validate = (schema, data, pluginId, stage) =>
+	Effect.gen(function* () {
+		const result = schema.safeParse(data);
+		if (result.success) {
+			return result.data;
+		}
+		return yield* Effect.fail(
+			new ValidationError({
+				pluginId,
+				stage,
+				zodError: result.error,
+			}),
+		);
+	});