@alpic80/rivet-core 1.19.1-aidon.1

package/dist/esm/utils/serialization/serialization_v4.js

@@ -0,0 +1,200 @@
+import { mapValues } from 'lodash-es';
+import stableStringify from 'safe-stable-stringify';
+import * as yaml from 'yaml';
+import { doubleCheckProject } from './serializationUtils.js';
+import { entries } from '../typeSafety.js';
+export function projectV4Deserializer(data) {
+    if (typeof data !== 'string') {
+        throw new Error('Project v4 deserializer requires a string');
+    }
+    const serializedProject = yaml.parse(data);
+    if (serializedProject.version !== 4) {
+        throw new Error('Project v4 deserializer requires a version 4 project');
+    }
+    const [project, attachedData] = fromSerializedProject(serializedProject.data);
+    doubleCheckProject(project);
+    return [project, attachedData];
+}
+export function graphV4Deserializer(data) {
+    if (typeof data !== 'string') {
+        throw new Error('Graph v4 deserializer requires a string');
+    }
+    const serializedGraph = yaml.parse(data);
+    if (serializedGraph.version !== 4) {
+        throw new Error('Graph v4 deserializer requires a version 4 graph');
+    }
+    return fromSerializedGraph(serializedGraph.data);
+}
+export function projectV4Serializer(project, attachedData) {
+    const filteredProject = {
+        ...project,
+        metadata: {
+            ...project.metadata,
+            path: undefined,
+        },
+    };
+    // Make sure all data is ordered deterministically first
+    const stabilized = JSON.parse(stableStringify(toSerializedProject(filteredProject, attachedData)));
+    const serialized = yaml.stringify({
+        version: 4,
+        data: stabilized,
+    }, null, {
+        indent: 2,
+    });
+    return serialized;
+}
+export function graphV4Serializer(graph) {
+    // Make sure all data is ordered deterministically first
+    const stabilized = JSON.parse(stableStringify(toSerializedGraph(graph)));
+    const serialized = yaml.stringify({
+        version: 4,
+        data: stabilized,
+    }, null, {
+        indent: 2,
+    });
+    return serialized;
+}
+function toSerializedProject(project, attachedData) {
+    return {
+        metadata: project.metadata,
+        graphs: mapValues(project.graphs, (graph) => toSerializedGraph(graph)),
+        attachedData,
+        plugins: project.plugins ?? [],
+    };
+}
+function fromSerializedProject(serializedProject) {
+    return [
+        {
+            metadata: serializedProject.metadata,
+            graphs: mapValues(serializedProject.graphs, (graph) => fromSerializedGraph(graph)),
+            plugins: serializedProject.plugins ?? [],
+        },
+        serializedProject.attachedData ?? {},
+    ];
+}
+function toSerializedGraph(graph) {
+    const graphMetadata = {
+        id: graph.metadata.id,
+        name: graph.metadata.name,
+        description: graph.metadata.description,
+    };
+    if (graph.metadata.attachedData) {
+        graphMetadata.attachedData = graph.metadata.attachedData;
+    }
+    return {
+        metadata: graphMetadata,
+        nodes: graph.nodes.reduce((acc, node) => ({
+            ...acc,
+            [getGraphNodeKey(node)]: toSerializedNode(node, graph.nodes, graph.connections),
+        }), {}),
+    };
+}
+function getGraphNodeKey(node) {
+    return `[${node.id}]:${node.type} "${node.title}"`;
+}
+function deserializeGraphNodeKey(key) {
+    const { nodeId, type, title } = key.match(/^\[(?<nodeId>[^\]]+)\]:(?<type>[^\s]+) "(?<title>.*)"$/)?.groups ?? {};
+    if (!nodeId || !type || !title) {
+        throw new Error(`Invalid graph node key: ${key}`);
+    }
+    return [nodeId, type, title];
+}
+function fromSerializedGraph(serializedGraph) {
+    const allConnections = [];
+    const allNodes = [];
+    for (const [serializedNodeInfo, node] of entries(serializedGraph.nodes)) {
+        const [chartNode, connections] = fromSerializedNode(node, serializedNodeInfo);
+        allNodes.push(chartNode);
+        allConnections.push(...connections);
+    }
+    const metadata = {
+        id: serializedGraph.metadata.id,
+        name: serializedGraph.metadata.name,
+        description: serializedGraph.metadata.description,
+    };
+    if (serializedGraph.metadata.attachedData) {
+        metadata.attachedData = serializedGraph.metadata.attachedData;
+    }
+    return {
+        metadata,
+        nodes: allNodes,
+        connections: allConnections,
+    };
+}
+function toSerializedNode(node, allNodes, allConnections) {
+    const outgoingConnections = allConnections
+        .filter((connection) => connection.outputNodeId === node.id)
+        .map((connection) => toSerializedConnection(connection, allNodes))
+        .sort();
+    return {
+        description: node.description?.trim() ? node.description : undefined,
+        visualData: `${node.visualData.x}/${node.visualData.y}/${node.visualData.width ?? 'null'}/${node.visualData.zIndex ?? 'null'}/${node.visualData.color?.border ?? ''}/${node.visualData.color?.bg ?? ''}`,
+        isSplitRun: node.isSplitRun ? true : undefined,
+        splitRunMax: node.isSplitRun ? node.splitRunMax : undefined,
+        isSplitSequential: node.isSplitSequential ? true : undefined,
+        data: Object.keys(node.data ?? {}).length > 0 ? node.data : undefined,
+        outgoingConnections: outgoingConnections.length > 0 ? outgoingConnections : undefined,
+        variants: (node.variants?.length ?? 0) > 0 ? node.variants : undefined,
+        disabled: node.disabled ? true : undefined,
+    };
+}
+function fromSerializedNode(serializedNode, serializedNodeInfo) {
+    const [nodeId, type, title] = deserializeGraphNodeKey(serializedNodeInfo);
+    const [x, y, width, zIndex, borderColor, bgColor] = serializedNode.visualData.split('/');
+    const connections = serializedNode.outgoingConnections?.map((serializedConnection) => fromSerializedConnection(serializedConnection, nodeId)) ?? [];
+    const color = borderColor || bgColor ? { border: borderColor, bg: bgColor } : undefined;
+    return [
+        {
+            id: nodeId,
+            type,
+            title,
+            description: serializedNode.description,
+            isSplitRun: serializedNode.isSplitRun ?? false,
+            splitRunMax: serializedNode.splitRunMax ?? 10,
+            isSplitSequential: serializedNode.isSplitSequential ?? false,
+            visualData: {
+                x: parseFloat(x),
+                y: parseFloat(y),
+                width: width === 'null' ? undefined : parseFloat(width),
+                zIndex: zIndex === 'null' ? undefined : parseFloat(zIndex),
+                color,
+            },
+            data: serializedNode.data ?? {},
+            variants: serializedNode.variants ?? [],
+            disabled: serializedNode.disabled,
+        },
+        connections,
+    ];
+}
+function toSerializedConnection(connection, allNodes) {
+    return `${connection.outputId}->"${allNodes.find((node) => node.id === connection.inputNodeId)?.title}" ${connection.inputNodeId}/${connection.inputId}`;
+}
+function fromSerializedConnection(connection, nodeId) {
+    try {
+        const [, outputId, , inputNodeId, inputId] = connection.match(/(.+)->"(.+)"\s+(.+)\/(.+)/);
+        return {
+            outputId: outputId,
+            outputNodeId: nodeId,
+            inputId: inputId,
+            inputNodeId: inputNodeId,
+        };
+    }
+    catch (err) {
+        throw new Error(`Invalid connection: ${connection}`);
+    }
+}
+export function datasetV4Serializer(datasets) {
+    const dataContainer = {
+        datasets,
+    };
+    const data = JSON.stringify(dataContainer);
+    return data;
+}
+export function datasetV4Deserializer(serializedDatasets) {
+    const stringData = serializedDatasets;
+    const dataContainer = JSON.parse(stringData);
+    if (!dataContainer.datasets) {
+        throw new Error('Invalid dataset data');
+    }
+    return dataContainer.datasets;
+}

package/dist/esm/utils/symbols.js

@@ -0,0 +1,2 @@
+export const Warnings = '__internalPort_Warnings';
+export const WarningsPort = Warnings;

package/dist/esm/utils/time.js

@@ -0,0 +1,14 @@
+/** Returns a number of milliseconds, strongly typed. */
+export const ms = (numMilliseconds) => numMilliseconds;
+/** Returns a number of seconds, as milliseconds. */
+export const seconds = (numSeconds) => ms(numSeconds * 1000);
+/** Returns a number of minutes, as milliseconds. */
+export const minutes = (numMinutes) => seconds(numMinutes * 60);
+/** Returns a number of hours, as milliseconds. */
+export const hours = (numHours) => minutes(numHours * 60);
+/** Returns a number of days, as milliseconds. */
+export const days = (numDays) => hours(numDays * 24);
+/** Returns a number of microseconds, strongly typed. To type a µ, press option+m on mac os. */
+export const µs = (numMicroseconds) => numMicroseconds;
+export const µsToMs = (numMicroseconds) => ms(numMicroseconds / 1000);
+export const msToµs = (numMilliseconds) => µs(numMilliseconds * 1000);

package/dist/esm/utils/typeSafety.js

@@ -0,0 +1,42 @@
+/**
+ * Type-safe version of {@link Object.entries} which can infer the type for the keys in the
+ * returned pairs when the input object has a branded type as its key.
+ *
+ * @example
+ * type MyId = string & { __myIdBrand: unknown };
+ * // Types check out!
+ * const things: [MyId, Thing][] = Objects.entries({} as Record<MyId, Thing>);
+ *
+ * @see {@link fromEntries}
+ */
+export const entries = (
+// In principle we should be able to use only Partial<> as the type is a superset of the non-
+// Partial type, but in practice this causes the compiler to infer a too-restricted type for V
+// in some cases involving unions.
+object) => (object == null ? [] : Object.entries(object));
+/**
+ * Type-safe version of {@link Object.fromEntries} which can infer the type for the key in the
+ * returned object when the input array has a branded type as its key.
+ *
+ * @example
+ * type MyId = string & { __myIdBrand: unknown };
+ * // Types check out!
+ * const things: Partial<Record<MyId, Thing>> = Objects.fromEntries([] as [MyId, Thing][]);
+ *
+ * @see {@link entries}
+ */
+// This returns a Partial<> because we can't guarantee that every key is present when K is a
+// string literal union (which is what Record<K, V> would be saying). As a side effect, this means
+// that using this with branded IDs will force null checks downstream, which is good practice.
+export function fromEntries(entries_) {
+    return Object.fromEntries(entries_);
+}
+export function keys(o) {
+    return Object.keys(o);
+}
+export function values(o) {
+    return Object.values(o);
+}
+export function mapValues(o, fn) {
+    return Object.fromEntries(Object.entries(o).map(([key, value]) => [key, fn(value)]));
+}

package/dist/types/api/createProcessor.d.ts

@@ -0,0 +1,37 @@
+import type { PascalCase } from 'type-fest';
+import { type AttachedData, type AudioProvider, type DataValue, type DatasetProvider, type ExternalFunction, type NativeApi, type NodeRegistration, type ProcessContext, type ProcessEvents, type Project, type RivetEventStreamFilterSpec, type Settings } from '../index.js';
+import { GraphProcessor } from '../model/GraphProcessor.js';
+export type LooseDataValue = DataValue | string | number | boolean;
+export type RunGraphOptions = {
+    graph?: string;
+    inputs?: Record<string, LooseDataValue>;
+    context?: Record<string, LooseDataValue>;
+    nativeApi?: NativeApi;
+    datasetProvider?: DatasetProvider;
+    audioProvider?: AudioProvider;
+    externalFunctions?: {
+        [key: string]: ExternalFunction;
+    };
+    onUserEvent?: {
+        [key: string]: (data: DataValue | undefined) => void;
+    };
+    abortSignal?: AbortSignal;
+    registry?: NodeRegistration;
+    getChatNodeEndpoint?: ProcessContext['getChatNodeEndpoint'];
+} & {
+    [P in keyof ProcessEvents as `on${PascalCase<P>}`]?: (params: ProcessEvents[P]) => void;
+} & Settings;
+export declare function looseDataValuesToDataValues(values: Record<string, LooseDataValue>): Record<string, DataValue>;
+export declare function looseDataValueToDataValue(value: LooseDataValue): DataValue;
+export declare function coreCreateProcessor(project: Project, options: RunGraphOptions): {
+    processor: GraphProcessor;
+    inputs: Record<string, DataValue>;
+    contextValues: Record<string, DataValue>;
+    getEvents: (spec: RivetEventStreamFilterSpec) => AsyncGenerator<import("./streaming.js").RivetEventStreamEventInfo, void, unknown>;
+    getSSEStream: (spec: RivetEventStreamFilterSpec) => ReadableStream<Uint8Array>;
+    streamNode: (nodeIdOrTitle: string) => ReadableStream<string>;
+    run(): Promise<import("../index.js").GraphOutputs>;
+};
+export declare function coreRunGraph(project: Project, options: RunGraphOptions): Promise<Record<string, DataValue>>;
+export declare function loadProjectFromString(content: string): Project;
+export declare function loadProjectAndAttachedDataFromString(content: string): [Project, AttachedData];

package/dist/types/api/streaming.d.ts

@@ -0,0 +1,56 @@
+import { type NodeId, type Inputs, type Outputs, type GraphOutputs, type GraphProcessor } from '../index.js';
+export type RivetEventStreamFilterSpec = {
+    /** Stream partial output deltas for the specified node IDs or node titles. */
+    partialOutputs?: string[] | true;
+    /** Send the graph output when done? */
+    done?: boolean;
+    /** If the graph errors, send an error event? */
+    error?: boolean;
+    /** Stream node start events for the specified node IDs or node titles. */
+    nodeStart?: string[] | true;
+    /** Stream node finish events for the specified nodeIDs or node titles. */
+    nodeFinish?: string[] | true;
+};
+/** Map of all possible event names to their data for streaming events. */
+export type RivetEventStreamEvent = {
+    /** Deltas for partial outputs. */
+    partialOutput: {
+        nodeId: NodeId;
+        nodeTitle: string;
+        delta: string;
+    };
+    nodeStart: {
+        nodeId: NodeId;
+        nodeTitle: string;
+        inputs: Inputs;
+    };
+    nodeFinish: {
+        nodeId: NodeId;
+        nodeTitle: string;
+        outputs: Outputs;
+    };
+    done: {
+        graphOutput: GraphOutputs;
+    };
+    error: {
+        error: string;
+    };
+};
+export type RivetEventStreamEventInfo = {
+    [P in keyof RivetEventStreamEvent]: {
+        type: P;
+    } & RivetEventStreamEvent[P];
+}[keyof RivetEventStreamEvent];
+/** A simplified way to listen and stream processor events, including filtering. */
+export declare function getProcessorEvents(processor: GraphProcessor, spec: RivetEventStreamFilterSpec): AsyncGenerator<RivetEventStreamEventInfo, void>;
+/**
+ * Creates a ReadableStream for processor events, following the Server-Sent Events protocol.
+ * https://developer.mozilla.org/en-US/docs/Web/API/EventSource
+ *
+ * Includes configuration for what events to send to the client, for example you can stream the partial output deltas
+ * for specific nodes, and/or the graph output when done.
+ */
+export declare function getProcessorSSEStream(processor: GraphProcessor, 
+/** The spec for what you're streaming to the client */
+spec: RivetEventStreamFilterSpec): ReadableStream<Uint8Array>;
+export declare function getSingleNodeStream(processor: GraphProcessor, nodeIdOrTitle: string): ReadableStream<string>;

package/dist/types/exports.d.ts

@@ -0,0 +1,33 @@
+export * from './utils/index.js';
+export * from './model/GraphProcessor.js';
+export * from './model/DataValue.js';
+export * from './model/NodeBase.js';
+export * from './model/NodeGraph.js';
+export * from './model/NodeImpl.js';
+export * from './model/NodeDefinition.js';
+export * from './model/Nodes.js';
+export * from './model/Project.js';
+export * from './native/BaseDir.js';
+export * from './native/NativeApi.js';
+export * from './native/BrowserNativeApi.js';
+export * from './model/ProcessContext.js';
+export * from './integrations/integrations.js';
+import './integrations/enableIntegrations.js';
+export * from './integrations/VectorDatabase.js';
+export * from './integrations/EmbeddingGenerator.js';
+export * from './integrations/LLMProvider.js';
+export * from './recording/ExecutionRecorder.js';
+export * from './recording/RecordedEvents.js';
+export * from './model/RivetPlugin.js';
+export * from './plugins.js';
+export * from './model/NodeRegistration.js';
+export type * from './model/Settings.js';
+export * from './model/EditorDefinition.js';
+export * from './model/NodeBodySpec.js';
+export * from './integrations/DatasetProvider.js';
+export * from './model/Dataset.js';
+export * from './api/streaming.js';
+export * from './api/createProcessor.js';
+export * from './integrations/AudioProvider.js';
+import * as openai from './utils/openai.js';
+export { openai };

package/dist/types/index.d.ts

@@ -0,0 +1,4 @@
+import type * as RivetTypes from './exports.js';
+export * from './exports.js';
+export declare const Rivet: typeof RivetTypes;
+export type RivetPluginInitializer = (rivet: typeof Rivet) => RivetTypes.RivetPlugin;

package/dist/types/integrations/AudioProvider.d.ts

@@ -0,0 +1,4 @@
+import type { AudioDataValue } from '../model/DataValue.js';
+export interface AudioProvider {
+    playAudio(audio: AudioDataValue, abort: AbortSignal): Promise<void>;
+}

package/dist/types/integrations/DatasetProvider.d.ts

@@ -0,0 +1,33 @@
+import { type Dataset, type DatasetId, type DatasetMetadata, type DatasetRow, type ProjectId } from '../index.js';
+import type { CombinedDataset } from '../utils/index.js';
+export interface DatasetProvider {
+    getDatasetMetadata(id: DatasetId): Promise<DatasetMetadata | undefined>;
+    getDatasetsForProject(projectId: ProjectId): Promise<DatasetMetadata[]>;
+    getDatasetData(id: DatasetId): Promise<Dataset>;
+    putDatasetData(id: DatasetId, data: Dataset): Promise<void>;
+    putDatasetRow(id: DatasetId, row: DatasetRow): Promise<void>;
+    putDatasetMetadata(metadata: DatasetMetadata): Promise<void>;
+    clearDatasetData(id: DatasetId): Promise<void>;
+    deleteDataset(id: DatasetId): Promise<void>;
+    /** Gets the K nearest neighbor rows to the given vector. */
+    knnDatasetRows(datasetId: DatasetId, k: number, vector: number[]): Promise<(DatasetRow & {
+        distance?: number;
+    })[]>;
+    exportDatasetsForProject(projectId: ProjectId): Promise<CombinedDataset[]>;
+}
+export declare class InMemoryDatasetProvider implements DatasetProvider {
+    #private;
+    constructor(datasets: CombinedDataset[]);
+    getDatasetMetadata(id: DatasetId): Promise<DatasetMetadata | undefined>;
+    getDatasetsForProject(projectId: ProjectId): Promise<DatasetMetadata[]>;
+    getDatasetData(id: DatasetId): Promise<Dataset>;
+    putDatasetRow(id: DatasetId, row: DatasetRow): Promise<void>;
+    putDatasetData(id: DatasetId, data: Dataset): Promise<void>;
+    putDatasetMetadata(metadata: DatasetMetadata): Promise<void>;
+    clearDatasetData(id: DatasetId): Promise<void>;
+    deleteDataset(id: DatasetId): Promise<void>;
+    knnDatasetRows(datasetId: DatasetId, k: number, vector: number[]): Promise<(DatasetRow & {
+        distance?: number | undefined;
+    })[]>;
+    exportDatasetsForProject(_projectId: ProjectId): Promise<CombinedDataset[]>;
+}

package/dist/types/integrations/EmbeddingGenerator.d.ts

@@ -0,0 +1,3 @@
+export interface EmbeddingGenerator {
+    generateEmbedding(text: string, options?: Record<string, unknown>): Promise<number[]>;
+}

package/dist/types/integrations/GptTokenizerTokenizer.d.ts

@@ -0,0 +1,20 @@
+import { type ChatMessage, type GptFunction } from '../index.js';
+import type { Tokenizer, TokenizerCallInfo } from './Tokenizer.js';
+import Emittery from 'emittery';
+export declare class GptTokenizerTokenizer implements Tokenizer {
+    emitter: Emittery<{
+        error: Error;
+    }, {
+        error: Error;
+    } & import("emittery").OmnipresentEventData, never>;
+    on(event: 'error', listener: (err: Error) => void): void;
+    getTokenCountForString(input: string, _info: TokenizerCallInfo): Promise<number>;
+    getTokenCountForMessages(messages: ChatMessage[], functions: GptFunction[] | undefined, _info: TokenizerCallInfo): Promise<number>;
+    /**
+     * Converts GPT Functions to approximate TypeScript-style string.
+     * Per thread: https://community.openai.com/t/how-to-calculate-the-tokens-when-using-function-call/266573/24
+     * We should consider using a different library, eg. https://github.com/hmarr/openai-chat-tokens
+     * @param functions
+     */
+    private convertGptFunctionsToPromptString;
+}

package/dist/types/integrations/LLMProvider.d.ts

@@ -0,0 +1,7 @@
+import { type ArrayDataValue, type ChatMessageDataValue } from '../model/DataValue.js';
+export type LLMParameters = {
+    [key: string]: string;
+};
+export interface LLMProvider {
+    getCompletions(messages: ArrayDataValue<ChatMessageDataValue>, parameters: LLMParameters): Promise<ArrayDataValue<ChatMessageDataValue>>;
+}

package/dist/types/integrations/Tokenizer.d.ts

@@ -0,0 +1,11 @@
+import type { ChartNode, ChatMessage, GptFunction } from '../index.js';
+export type TokenizerCallInfo = {
+    node: ChartNode;
+    model?: string;
+    endpoint?: string;
+};
+export type Tokenizer = {
+    on(event: 'error', listener: (err: Error) => void): void;
+    getTokenCountForString(input: string, info: TokenizerCallInfo): Promise<number>;
+    getTokenCountForMessages(messages: ChatMessage[], gptFunctions: GptFunction[] | undefined, info: TokenizerCallInfo): Promise<number>;
+};

package/dist/types/integrations/VectorDatabase.d.ts

@@ -0,0 +1,7 @@
+import { type DataValue, type VectorDataValue, type ArrayDataValue, type ScalarDataValue } from '../model/DataValue.js';
+export interface VectorDatabase {
+    store(collection: DataValue, vector: VectorDataValue, data: DataValue, metadata: {
+        id?: string;
+    }): Promise<void>;
+    nearestNeighbors(collection: DataValue, vector: VectorDataValue, k: number): Promise<ArrayDataValue<ScalarDataValue>>;
+}

package/dist/types/integrations/enableIntegrations.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/types/integrations/integrations.d.ts

@@ -0,0 +1,12 @@
+import { type InternalProcessContext } from '../model/ProcessContext.js';
+import { type EmbeddingGenerator } from './EmbeddingGenerator.js';
+import { type LLMProvider } from './LLMProvider.js';
+import { type VectorDatabase } from './VectorDatabase.js';
+export type IntegrationFactories = {
+    vectorDatabase: (context: InternalProcessContext) => VectorDatabase;
+    llmProvider: (context: InternalProcessContext) => LLMProvider;
+    embeddingGenerator: (context: InternalProcessContext) => EmbeddingGenerator;
+};
+export type IntegrationType = keyof IntegrationFactories;
+export declare function registerIntegration<T extends IntegrationType>(type: T, integrationKey: string, factory: IntegrationFactories[T]): void;
+export declare function getIntegration<T extends IntegrationType>(type: T, integrationKey: string, context: InternalProcessContext): ReturnType<IntegrationFactories[T]>;

package/dist/types/integrations/openai/OpenAIEmbeddingGenerator.d.ts

@@ -0,0 +1,10 @@
+import { type Settings } from '../../index.js';
+import { type EmbeddingGenerator } from '../EmbeddingGenerator.js';
+import { OpenAI } from 'openai';
+type OpenAIOptions = Pick<OpenAI.EmbeddingCreateParams, 'model' | 'dimensions'>;
+export declare class OpenAIEmbeddingGenerator implements EmbeddingGenerator {
+    #private;
+    constructor(settings: Settings);
+    generateEmbedding(text: string, options?: OpenAIOptions): Promise<number[]>;
+}
+export {};