@alpic80/rivet-core 1.19.1-aidon.1

package/dist/esm/plugins/autoevals/AutoEvalsNode.js

@@ -0,0 +1,223 @@
+import { nanoid } from 'nanoid/non-secure';
+import { dedent } from 'ts-dedent';
+import { Battle, ClosedQA, Factuality, Humor, Possible, Security, Summary, Translation, Sql, } from 'autoevals';
+import { match } from 'ts-pattern';
+import { coerceType } from '../../utils/coerceType.js';
+import { pluginNodeDefinition } from '../../model/NodeDefinition.js';
+const options = [
+    { label: 'Factuality', value: 'factuality' },
+    { label: 'Humor', value: 'humor' },
+    { label: 'Security', value: 'security' },
+    { label: 'Possible', value: 'possible' },
+    { label: 'Summary', value: 'summary' },
+    { label: 'Translation', value: 'translation' },
+    { label: 'Battle', value: 'battle' },
+    { label: 'Closed Q&A', value: 'closed_q_a' },
+    { label: 'SQL', value: 'sql' },
+];
+export const AutoEvalsNodeImpl = {
+    create() {
+        const chartNode = {
+            type: 'autoevals',
+            title: 'Autoevals',
+            id: nanoid(),
+            visualData: {
+                x: 0,
+                y: 0,
+                width: 250,
+            },
+            data: {
+                evaluatorName: 'factuality',
+            },
+        };
+        return chartNode;
+    },
+    getInputDefinitions(data) {
+        const base = [
+            {
+                id: 'output',
+                dataType: 'string',
+                title: 'Output',
+            },
+            {
+                id: 'expected',
+                dataType: 'string',
+                title: 'Expected',
+            },
+        ];
+        const forEvaluator = match(data.evaluatorName)
+            .with('factuality', () => [
+            {
+                id: 'input',
+                dataType: 'string',
+                title: 'Input',
+            },
+        ])
+            .with('battle', () => [
+            {
+                id: 'instructions',
+                dataType: 'string',
+                title: 'Instructions',
+            },
+        ])
+            .with('closed_q_a', () => [
+            {
+                id: 'input',
+                dataType: 'string',
+                title: 'Input',
+            },
+            {
+                id: 'criteria',
+                dataType: 'string',
+                title: 'Criteria',
+            },
+        ])
+            .with('humor', () => [])
+            .with('possible', () => [
+            {
+                id: 'input',
+                dataType: 'string',
+                title: 'Input',
+            },
+        ])
+            .with('security', () => [])
+            .with('summary', () => [
+            {
+                id: 'input',
+                dataType: 'string',
+                title: 'Input',
+            },
+        ])
+            .with('translation', () => [
+            {
+                id: 'input',
+                dataType: 'string',
+                title: 'Input',
+            },
+            {
+                id: 'language',
+                dataType: 'string',
+                title: 'Language',
+            },
+        ])
+            .with('sql', () => [
+            {
+                id: 'input',
+                dataType: 'string',
+                title: 'Input',
+            },
+        ])
+            .with(undefined, () => [])
+            .exhaustive();
+        return [...forEvaluator, ...base];
+    },
+    getOutputDefinitions() {
+        return [
+            {
+                dataType: 'number',
+                id: 'score',
+                title: 'Score',
+            },
+            {
+                dataType: 'string',
+                id: 'rationale',
+                title: 'Rationale',
+            },
+            {
+                dataType: 'object',
+                id: 'metadata',
+                title: 'Metadata',
+            },
+        ];
+    },
+    getEditors() {
+        return [
+            {
+                type: 'dropdown',
+                dataKey: 'evaluatorName',
+                label: 'Evaluator',
+                options,
+            },
+        ];
+    },
+    getBody(data) {
+        return options.find((option) => option.value === data.evaluatorName)?.label ?? 'None';
+    },
+    getUIData() {
+        return {
+            infoBoxBody: dedent `
+        Evaluates the validity of a response using the autoevals library.
+      `,
+            infoBoxTitle: 'Autoevals Node',
+            contextMenuTitle: 'Autoevals',
+            group: 'Custom',
+        };
+    },
+    async process(data, inputs, context) {
+        const evaluatorName = data.evaluatorName;
+        const output = coerceType(inputs['output'], 'string');
+        const expected = coerceType(inputs['expected'], 'string');
+        const baseArgs = {
+            output,
+            expected,
+            openAiApiKey: context.settings.openAiKey,
+            openAiOrganizationId: context.settings.openAiOrganization,
+        };
+        const result = await match(evaluatorName)
+            .with('factuality', () => {
+            const input = coerceType(inputs['input'], 'string');
+            return Factuality({ ...baseArgs, input });
+        })
+            .with('battle', () => {
+            const instructions = coerceType(inputs['instructions'], 'string');
+            return Battle({ ...baseArgs, instructions });
+        })
+            .with('closed_q_a', () => {
+            const input = coerceType(inputs['input'], 'string');
+            const criteria = coerceType(inputs['criteria'], 'string');
+            return ClosedQA({ ...baseArgs, input, criteria });
+        })
+            .with('humor', () => {
+            return Humor({ ...baseArgs });
+        })
+            .with('possible', () => {
+            const input = coerceType(inputs['input'], 'string');
+            return Possible({ ...baseArgs, input });
+        })
+            .with('security', () => {
+            return Security({ ...baseArgs });
+        })
+            .with('summary', () => {
+            const input = coerceType(inputs['input'], 'string');
+            return Summary({ ...baseArgs, input });
+        })
+            .with('translation', () => {
+            const input = coerceType(inputs['input'], 'string');
+            const language = coerceType(inputs['language'], 'string');
+            return Translation({ ...baseArgs, input, language });
+        })
+            .with('sql', () => {
+            const input = coerceType(inputs['input'], 'string');
+            return Sql({ ...baseArgs, input });
+        })
+            .with(undefined, () => {
+            throw new Error('Evaluator name is undefined');
+        })
+            .exhaustive();
+        return {
+            ['score']: {
+                type: 'number',
+                value: result.score,
+            },
+            ['rationale']: {
+                type: 'string',
+                value: result.metadata?.rationale ?? '',
+            },
+            ['metadata']: {
+                type: 'object',
+                value: result.metadata,
+            },
+        };
+    },
+};
+export const autoEvalsNode = pluginNodeDefinition(AutoEvalsNodeImpl, 'Autoevals');

package/dist/esm/plugins/autoevals/index.js

@@ -0,0 +1,2 @@
+import { autoevalsPlugin } from './plugin.js';
+export default autoevalsPlugin;

package/dist/esm/plugins/autoevals/plugin.js

@@ -0,0 +1,8 @@
+import { autoEvalsNode } from './AutoEvalsNode.js';
+export const autoevalsPlugin = {
+    id: 'autoevals',
+    name: 'Autoevals',
+    register: (register) => {
+        register(autoEvalsNode);
+    },
+};

package/dist/esm/plugins/gentrace/index.js

@@ -0,0 +1,2 @@
+import { gentracePlugin } from './plugin.js';
+export default gentracePlugin;

package/dist/esm/plugins/gentrace/plugin.js

@@ -0,0 +1,192 @@
+import { Pipeline, StepRun, init, runTest, getPipelines } from '@gentrace/core';
+import {} from '../../index.js';
+import { mapValues } from 'lodash-es';
+import { ExecutionRecorder } from '../../recording/ExecutionRecorder.js';
+import { inferType } from '../../utils/coerceType.js';
+import { GraphProcessor } from '../../model/GraphProcessor.js';
+const apiKeyConfigSpec = {
+    type: 'secret',
+    label: 'Gentrace API Key',
+    description: 'The API key for the Gentrace service.',
+    pullEnvironmentVariable: 'GENTRACE_API_KEY',
+    helperText: 'Create at https://gentrace.ai/settings/api-keys',
+};
+function initializeGentrace(gentraceApiKey) {
+    init({
+        apiKey: gentraceApiKey,
+    });
+}
+export const runGentraceTests = async (gentracePipelineSlug, settings, project, graph, nativeApi) => {
+    const gentraceApiKey = settings.pluginSettings?.gentrace?.gentraceApiKey;
+    if (!gentraceApiKey) {
+        throw new Error('Gentrace API key not set.');
+    }
+    const graphId = graph.metadata?.id;
+    if (!graphId) {
+        throw new Error('Graph ID not set.');
+    }
+    initializeGentrace(gentraceApiKey);
+    const response = await runTest(gentracePipelineSlug, async (testCase) => {
+        const pipeline = new Pipeline({
+            slug: gentracePipelineSlug,
+        });
+        const runner = pipeline.start();
+        const rivetFormattedInputs = mapValues(testCase.inputs, inferType);
+        const tempProject = {
+            ...project,
+            graphs: {
+                ...project.graphs,
+                [graph.metadata.id]: graph,
+            },
+        };
+        const recorder = new ExecutionRecorder();
+        const processor = new GraphProcessor(tempProject, graphId);
+        processor.executor = 'browser';
+        recorder.record(processor);
+        await processor.processGraph({
+            settings,
+            nativeApi,
+        }, rivetFormattedInputs);
+        const fullRecording = recorder.getRecording();
+        const stepRuns = convertRecordingToStepRuns(fullRecording, project, graphId);
+        stepRuns.forEach((stepRun) => {
+            runner.addStepRunNode(stepRun);
+        });
+        if (stepRuns.length === 0) {
+            throw new Error('No Rivet steps found. You need operations which are not Graph Input or Graph Output nodes.');
+        }
+        return ['', runner];
+    });
+    return response;
+};
+export const runRemoteGentraceTests = async (gentracePipelineSlug, settings, project, graph, runAndRecord) => {
+    const gentraceApiKey = settings.pluginSettings?.gentrace?.gentraceApiKey;
+    if (!gentraceApiKey) {
+        throw new Error('Gentrace API key not set.');
+    }
+    const graphId = graph.metadata?.id;
+    if (!graphId) {
+        throw new Error('Graph ID not set.');
+    }
+    initializeGentrace(gentraceApiKey);
+    const response = await runTest(gentracePipelineSlug, async (testCase) => {
+        const pipeline = new Pipeline({
+            slug: gentracePipelineSlug,
+        });
+        const runner = pipeline.start();
+        const rivetFormattedInputs = mapValues(testCase.inputs, inferType);
+        const fullRecording = await runAndRecord(rivetFormattedInputs);
+        const stepRuns = convertRecordingToStepRuns(fullRecording, project, graphId);
+        stepRuns.forEach((stepRun) => {
+            runner.addStepRunNode(stepRun);
+        });
+        if (stepRuns.length === 0) {
+            throw new Error('No Rivet steps found. You need operations which are not Graph Input or Graph Output nodes.');
+        }
+        return ['', runner];
+    });
+    return response;
+};
+function convertRecordingToStepRuns(recording, project, graphId) {
+    const partialProcessStartEndPairs = {};
+    recording.events.forEach((event) => {
+        const eventType = event?.type;
+        if (!eventType) {
+            return;
+        }
+        if (eventType === 'nodeStart' || eventType === 'nodeFinish') {
+            const processId = event?.data?.processId;
+            const nodeId = event?.data?.nodeId;
+            if (!processId) {
+                return;
+            }
+            let existingPair = partialProcessStartEndPairs[processId];
+            if (!existingPair) {
+                existingPair = {};
+                partialProcessStartEndPairs[processId] = existingPair;
+            }
+            existingPair.nodeId = nodeId;
+            if (eventType === 'nodeStart') {
+                existingPair.start = event.ts;
+                existingPair.inputs = event.data.inputs;
+            }
+            else {
+                existingPair.end = event.ts;
+                existingPair.outputs = event.data.outputs;
+            }
+        }
+    });
+    const processStartEndPairs = partialProcessStartEndPairs;
+    const selectedGraph = project.graphs[graphId];
+    if (!selectedGraph) {
+        return [];
+    }
+    // Convert to step runs
+    const stepRuns = [];
+    for (const [, pair] of Object.entries(processStartEndPairs)) {
+        const { nodeId } = pair;
+        const relatedNode = selectedGraph.nodes.find((node) => node.id === nodeId);
+        const nodeType = relatedNode?.type;
+        if (!nodeType || !relatedNode.data || nodeType === 'graphInput' || nodeType === 'graphOutput') {
+            continue;
+        }
+        const nodeData = relatedNode.data;
+        if (relatedNode) {
+            pair.modelParams = { ...nodeData, ...{ type: nodeType } };
+        }
+        if (nodeType === 'chat') {
+            const modelName = nodeData.model ? nodeData.model : '';
+            if (modelName.startsWith('gpt')) {
+                // Convert to OpenAI Gentrace node
+                const gentraceOpenAIInputs = { ...pair.inputs };
+                gentraceOpenAIInputs.messages = [
+                    {
+                        content: pair.inputs.prompt.value,
+                        role: 'user',
+                    },
+                ];
+                const gentraceOpenAIModelParams = { ...pair.modelParams };
+                gentraceOpenAIModelParams.model = modelName;
+                gentraceOpenAIModelParams.frequency_penalty = pair.modelParams.frequencyPenalty || null;
+                gentraceOpenAIModelParams.max_tokens = pair.modelParams.maxTokens || undefined;
+                gentraceOpenAIModelParams.presence_penalty = pair.modelParams.presencePenalty || null;
+                gentraceOpenAIModelParams.stop = pair.modelParams.stop || null;
+                gentraceOpenAIModelParams.temperature = pair.modelParams.temperature || null;
+                gentraceOpenAIModelParams.top_p = pair.modelParams.top_p || null;
+                const gentraceOpenAIOutputs = { ...pair.outputs };
+                const outputValues = Array.isArray(pair.outputs.response.value)
+                    ? pair.outputs.response.value
+                    : [pair.outputs.response.value];
+                gentraceOpenAIOutputs.choices = outputValues.map((outputValue, index) => {
+                    return {
+                        index,
+                        message: {
+                            content: outputValue,
+                            role: 'assistant',
+                        },
+                        usage: {
+                            completion_tokens: pair.outputs.responseTokens.value,
+                            prompt_tokens: pair.outputs.requestTokens.value,
+                            total_tokens: pair.outputs.responseTokens.value + pair.outputs.requestTokens.value,
+                        },
+                    };
+                });
+                stepRuns.push(new StepRun('openai', 'openai_createChatCompletion', pair.end - pair.start, new Date(pair.start).toISOString(), new Date(pair.end).toISOString(), gentraceOpenAIInputs, gentraceOpenAIModelParams, gentraceOpenAIOutputs, {}));
+                continue;
+            }
+        }
+        stepRuns.push(new StepRun('rivet', nodeType ? `rivet_operation_${nodeType}` : 'rivet_operation', pair.end - pair.start, new Date(pair.start).toISOString(), new Date(pair.end).toISOString(), pair.inputs, pair.modelParams, pair.outputs, {}));
+    }
+    return stepRuns;
+}
+export const getGentracePipelines = async (gentraceApiKey) => {
+    initializeGentrace(gentraceApiKey);
+    return await getPipelines();
+};
+export const gentracePlugin = {
+    id: 'gentrace',
+    name: 'Gentrace',
+    configSpec: {
+        gentraceApiKey: apiKeyConfigSpec,
+    },
+};

package/dist/esm/plugins/google/google.js

@@ -0,0 +1,60 @@
+export const googleModels = {
+    'gemini-pro': {
+        maxTokens: 32760,
+        cost: {
+            prompt: NaN,
+            completion: NaN,
+        },
+        displayName: 'Gemini Pro',
+    },
+    'gemini-pro-vision': {
+        maxTokens: 16384,
+        cost: {
+            prompt: NaN,
+            completion: NaN,
+        },
+        displayName: 'Gemini Pro Vision',
+    },
+};
+export const googleModelOptions = Object.entries(googleModels).map(([id, { displayName }]) => ({
+    value: id,
+    label: displayName,
+}));
+export async function* streamChatCompletions({ project, location, applicationCredentials, model, signal, max_output_tokens, temperature, top_p, top_k, prompt, }) {
+    const defaultSignal = new AbortController().signal;
+    // If you import normally, the Google auth library throws a fit.
+    const { VertexAI } = await import('@google-cloud/vertexai');
+    // Can't find a way to pass the credentials path in
+    process.env['GOOGLE_APPLICATION_CREDENTIALS'] = applicationCredentials;
+    const vertexAi = new VertexAI({ project, location });
+    const generativeModel = vertexAi.preview.getGenerativeModel({
+        model,
+        generation_config: {
+            max_output_tokens,
+            temperature,
+            top_p,
+            top_k,
+        },
+    });
+    const response = await generativeModel.generateContentStream({
+        contents: prompt,
+    });
+    let hadChunks = false;
+    for await (const chunk of response.stream) {
+        console.log('streaming google responses');
+        hadChunks = true;
+        if (!signal?.aborted && chunk.candidates[0]?.content.parts[0]?.text) {
+            yield {
+                completion: chunk.candidates[0]?.content.parts[0]?.text,
+                finish_reason: chunk.candidates[0]?.finishReason,
+                model,
+            };
+        }
+        else {
+            return;
+        }
+    }
+    if (!hadChunks) {
+        throw new Error(`No chunks received.`);
+    }
+}

package/dist/esm/plugins/google/index.js

@@ -0,0 +1,2 @@
+import { googlePlugin } from './plugin.js';
+export default googlePlugin;