openlayer 0.1.11 → 0.1.13

package/README.md

@@ -0,0 +1,35 @@
+<div align="left">
+  <img src="static/logo.png"><br>
+</div>
+
+# Openlayer | JavaScript/TypeScript Library
+
+[![npm version](https://badge.fury.io/js/openlayer.svg)](https://badge.fury.io/js/openlayer)
+
+## What is it?
+
+Openlayer is a debugging workspace for ML & Data Science. Openlayer combines and builds upon SOTA techniques in explainability, model and dataset versioning, synthetic data generation, data-centric testing and much more to form a powerful, **unified platform for model development**.
+
+👉 [Join our Discord community!](https://discord.gg/t6wS2g6MMB) We'd love to meet you and help you get started with Openlayer!
+
+This is the official JavaScript/TypeScript library for interacting with the Openlayer platform. Navigate [here](https://docs.openlayer.com) for a quickstart guide and for in-depth tutorials.
+
+## Main Features
+
+This library's primary function is to enable you to easily package your models and datasets and add them to your Openlayer account.
+
+## Installation
+
+Install with npm
+
+```console
+npm i openlayer
+```
+
+## Documentation
+
+The official documentation for this library can be found [here](https://docs.openlayer.com).
+
+## Contributing
+
+All contributions, bug reports, bug fixes, documentation improvements, enhancements, and ideas are welcome! Just send us a message on [Discord](https://discord.gg/t6wS2g6MMB).

package/dist/index.d.ts

@@ -1,16 +1,12 @@
 import { RequestOptions } from 'openai/core';
-import { ChatCompletion, ChatCompletionChunk, ChatCompletionCreateParams, Completion, CompletionCreateParams } from 'openai/resources';
+import { ChatCompletion, ChatCompletionChunk, ChatCompletionCreateParams, ChatCompletionMessageParam, Completion, CompletionCreateParams } from 'openai/resources';
 import { Stream } from 'openai/streaming';
 /**
  * Represents the data structure for a chat completion.
+ * Object keys represent a column name and the values represent the column value.
  */
-export interface ChatCompletionData {
-    [key: string]: any;
-    /**
-     * The input data for the chat completion. It can be a string, an array of strings,
-     * or a nested array of numbers.
-     */
-    input: string | string[] | number[] | number[][];
+export interface StreamingData {
+    [columnName: string]: any;
     /**
      * The latency of the chat completion in milliseconds. Optional.
      */
@@ -31,7 +27,7 @@ export interface ChatCompletionData {
 /**
  * Configuration settings for uploading chat completion data to Openlayer.
  */
-export interface ChatCompletionConfig {
+interface StreamingDataConfig {
     /**
      * The name of the column that stores the ground truth data. Can be null.
      */
@@ -43,7 +39,7 @@ export interface ChatCompletionConfig {
     /**
      * An array of names for input variable columns. Can be null.
      */
-    inputVariableNames: string[] | null;
+    inputVariableNames?: string[] | null;
     /**
      * The name of the column that stores latency data. Can be null.
      */
@@ -56,6 +52,10 @@ export interface ChatCompletionConfig {
      * The name of the column that stores output data. Can be null.
      */
     outputColumnName: string | null;
+    /**
+     * The full prompt history for the chat completion.
+     */
+    prompt?: ChatCompletionMessageParam[];
     /**
      * The name of the column that stores timestamp data. Can be null.
      */
@@ -129,45 +129,55 @@ type OpenlayerSampleVolumeGraph = {
 type OpenlayerTaskType = 'llm-base' | 'tabular-classification' | 'tabular-regression' | 'text-classification';
 export declare class OpenlayerClient {
     private openlayerApiKey?;
-    private openlayerDefaultDataConfig;
+    defaultConfig: StreamingDataConfig;
     private openlayerServerUrl;
     private version;
+    /**
+     * Constructs an OpenlayerClient instance.
+     * @param {OpenlayerClientConstructorProps} props - The config for the Openlayer client. The API key is required.
+     * @throws {Error} Throws an error if the Openlayer API key is not provided.
+     */
     constructor({ openlayerApiKey, openlayerServerUrl, }: OpenlayerClientConstructorProps);
     private resolvedQuery;
     /**
-     * Uploads data to Openlayer. This function takes ChatCompletionData and an optional
-     * ChatCompletionConfig, then uploads the data to the specified Openlayer Inference Pipeline.
-     * @param data The ChatCompletionData to be uploaded.
-     * @param config Configuration for the data upload.
-     * @throws Throws an error if Openlayer API key or project name are not set.
-     * @returns A promise that resolves when the data has been successfully uploaded.
+     * Streams data to the Openlayer inference pipeline.
+     * @param {StreamingData} data - The chat completion data to be streamed.
+     * @param {string} inferencePipelineId - The ID of the Openlayer inference pipeline to which data is streamed.
+     * @returns {Promise<void>} A promise that resolves when the data has been successfully streamed.
+     * @throws {Error} Throws an error if the Openlayer API key is not set or an error occurs in the streaming process.
      */
-    streamData: (data: ChatCompletionData, inferencePipelineId: string) => Promise<void>;
+    streamData: (data: StreamingData, config: StreamingDataConfig, inferencePipelineId: string) => Promise<void>;
     /**
-     * Creates a new inference pipeline in Openlayer, or loads an existing one if it already exists.
-     * @param name The name of the inference pipeline.
-     * @param projectId The project ID containing the inference pipeline.
-     * @throws Throws an error if the inference pipeline cannot be created or found.
-     * @returns A promise that resolves to an OpenlayerInferencePipeline object.
+     * Creates a new inference pipeline in Openlayer or loads an existing one.
+     * @param {string} projectId - The ID of the project containing the inference pipeline.
+     * @param {string} [name='production'] - The name of the inference pipeline, defaults to 'production'.
+     * @returns {Promise<OpenlayerInferencePipeline>} A promise that resolves to an OpenlayerInferencePipeline object.
+     * @throws {Error} Throws an error if the inference pipeline cannot be created or found.
      */
     createInferencePipeline: (projectId: string, name?: string) => Promise<OpenlayerInferencePipeline>;
     /**
-     * Creates a new project in Openlayer, or loads an existing one if it already exists.
-     * @param name The name of the project.
-     * @param taskType The type of task associated with the project.
-     * @param description Optional description of the project.
-     * @throws Throws an error if the project cannot be created or found.
-     * @returns A promise that resolves to an OpenlayerProject object.
+     * Creates a new project in Openlayer or loads an existing one.
+     * @param {string} name - The name of the project.
+     * @param {OpenlayerTaskType} taskType - The type of task associated with the project.
+     * @param {string} [description] - Optional description of the project.
+     * @returns {Promise<OpenlayerProject>} A promise that resolves to an OpenlayerProject object.
+     * @throws {Error} Throws an error if the project cannot be created or found.
      */
     createProject: (name: string, taskType: OpenlayerTaskType, description?: string) => Promise<OpenlayerProject>;
     /**
      * Loads an existing inference pipeline from Openlayer based on its name and project ID.
-     * @param name The name of the inference pipeline.
-     * @param projectId The project ID containing the inference pipeline.
-     * @throws Throws an error if the inference pipeline is not found.
-     * @returns A promise that resolves to an OpenlayerInferencePipeline object.
+     * @param {string} projectId - The ID of the project containing the inference pipeline.
+     * @param {string} [name='production'] - The name of the inference pipeline, defaults to 'production'.
+     * @returns {Promise<OpenlayerInferencePipeline>} A promise that resolves to an OpenlayerInferencePipeline object.
+     * @throws {Error} Throws an error if the inference pipeline is not found.
      */
     loadInferencePipeline: (projectId: string, name?: string) => Promise<OpenlayerInferencePipeline>;
+    /**
+     * Loads an existing project from Openlayer based on its name.
+     * @param {string} name - The name of the project.
+     * @returns {Promise<OpenlayerProject>} A promise that resolves to an OpenlayerProject object.
+     * @throws {Error} Throws an error if the project is not found.
+     */
     loadProject: (name: string) => Promise<OpenlayerProject>;
 }
 export declare class OpenAIMonitor {
@@ -176,24 +186,26 @@ export declare class OpenAIMonitor {
     private openlayerProjectName;
     private openlayerInferencePipelineName;
     private monitoringOn;
+    /**
+     * Constructs an OpenAIMonitor instance.
+     * @param {OpenAIMonitorConstructorProps} props - The configuration properties for the OpenAI and Openlayer clients.
+     */
     constructor({ openAiApiKey, openlayerApiKey, openlayerProjectName, openlayerInferencePipelineName, openlayerServerUrl, }: OpenAIMonitorConstructorProps);
     private formatChatCompletionInput;
     /**
-     * Creates a new ChatCompletion instance. If monitoring is not active, an error is thrown.
-     * This function also measures latency and uploads data to Openlayer.
-     * @param body The parameters for creating a chat completion.
-     * @param options Optional request options.
-     * @throws Throws an error if monitoring is not active or if there is no output received from OpenAI.
-     * @returns A promise that resolves to a ChatCompletion or a Stream of ChatCompletionChunks.
+     * Creates a chat completion using the OpenAI client and streams the result to Openlayer.
+     * @param {ChatCompletionCreateParams} body - The parameters for creating a chat completion.
+     * @param {RequestOptions} [options] - Optional request options.
+     * @returns {Promise<ChatCompletion | Stream<ChatCompletionChunk>>} Promise of a ChatCompletion or a Stream
+     * @throws {Error} Throws an error if monitoring is not active or if no output is received from OpenAI.
      */
     createChatCompletion: (body: ChatCompletionCreateParams, options?: RequestOptions) => Promise<ChatCompletion | Stream<ChatCompletionChunk>>;
     /**
-     * Creates a new Completion instance. If monitoring is not active, an error is thrown.
-     * This function also measures latency and uploads data to Openlayer.
-     * @param body The parameters for creating a completion.
-     * @param options Optional request options.
-     * @throws Throws an error if monitoring is not active or if no prompt is provided.
-     * @returns A promise that resolves to a Completion or a Stream of Completions.
+     * Creates a completion using the OpenAI client and streams the result to Openlayer.
+     * @param {CompletionCreateParams} body - The parameters for creating a completion.
+     * @param {RequestOptions} [options] - Optional request options.
+     * @returns {Promise<Completion | Stream<Completion>>} Promise that resolves to a Completion or a Stream.
+     * @throws {Error} Throws an error if monitoring is not active or if no prompt is provided.
      */
     createCompletion: (body: CompletionCreateParams, options?: RequestOptions) => Promise<Completion | Stream<Completion>>;
     /**

package/dist/index.js

@@ -21,11 +21,15 @@ const openai_1 = require("openai");
 const uuid_1 = require("uuid");
 const request_1 = require("./utils/request");
 class OpenlayerClient {
+    /**
+     * Constructs an OpenlayerClient instance.
+     * @param {OpenlayerClientConstructorProps} props - The config for the Openlayer client. The API key is required.
+     * @throws {Error} Throws an error if the Openlayer API key is not provided.
+     */
     constructor({ openlayerApiKey, openlayerServerUrl, }) {
-        this.openlayerDefaultDataConfig = {
+        this.defaultConfig = {
             groundTruthColumnName: null,
             inferenceIdColumnName: 'id',
-            inputVariableNames: ['input'],
             latencyColumnName: 'latency',
             numOfTokenColumnName: 'tokens',
             outputColumnName: 'output',
@@ -35,14 +39,13 @@ class OpenlayerClient {
         this.version = '0.1.0a16';
         this.resolvedQuery = (endpoint, args = {}) => (0, request_1.resolvedQuery)(this.openlayerServerUrl, endpoint, args);
         /**
-         * Uploads data to Openlayer. This function takes ChatCompletionData and an optional
-         * ChatCompletionConfig, then uploads the data to the specified Openlayer Inference Pipeline.
-         * @param data The ChatCompletionData to be uploaded.
-         * @param config Configuration for the data upload.
-         * @throws Throws an error if Openlayer API key or project name are not set.
-         * @returns A promise that resolves when the data has been successfully uploaded.
+         * Streams data to the Openlayer inference pipeline.
+         * @param {StreamingData} data - The chat completion data to be streamed.
+         * @param {string} inferencePipelineId - The ID of the Openlayer inference pipeline to which data is streamed.
+         * @returns {Promise<void>} A promise that resolves when the data has been successfully streamed.
+         * @throws {Error} Throws an error if the Openlayer API key is not set or an error occurs in the streaming process.
          */
-        this.streamData = (data, inferencePipelineId) => __awaiter(this, void 0, void 0, function* () {
+        this.streamData = (data, config, inferencePipelineId) => __awaiter(this, void 0, void 0, function* () {
             var _a;
             if (!this.openlayerApiKey) {
                 throw new Error('Openlayer API key are required for streaming data.');
@@ -52,7 +55,7 @@ class OpenlayerClient {
                 const dataStreamQuery = this.resolvedQuery(dataStreamEndpoint);
                 const response = yield fetch(dataStreamQuery, {
                     body: JSON.stringify({
-                        config: this.openlayerDefaultDataConfig,
+                        config,
                         rows: [
                             Object.assign(Object.assign({}, data), { id: (0, uuid_1.v4)(), timestamp: Math.round(((_a = data.timestamp) !== null && _a !== void 0 ? _a : Date.now()) / 1000) }),
                         ],
@@ -75,11 +78,11 @@ class OpenlayerClient {
             }
         });
         /**
-         * Creates a new inference pipeline in Openlayer, or loads an existing one if it already exists.
-         * @param name The name of the inference pipeline.
-         * @param projectId The project ID containing the inference pipeline.
-         * @throws Throws an error if the inference pipeline cannot be created or found.
-         * @returns A promise that resolves to an OpenlayerInferencePipeline object.
+         * Creates a new inference pipeline in Openlayer or loads an existing one.
+         * @param {string} projectId - The ID of the project containing the inference pipeline.
+         * @param {string} [name='production'] - The name of the inference pipeline, defaults to 'production'.
+         * @returns {Promise<OpenlayerInferencePipeline>} A promise that resolves to an OpenlayerInferencePipeline object.
+         * @throws {Error} Throws an error if the inference pipeline cannot be created or found.
          */
         this.createInferencePipeline = (projectId, name = 'production') => __awaiter(this, void 0, void 0, function* () {
             try {
@@ -107,12 +110,12 @@ class OpenlayerClient {
             }
         });
         /**
-         * Creates a new project in Openlayer, or loads an existing one if it already exists.
-         * @param name The name of the project.
-         * @param taskType The type of task associated with the project.
-         * @param description Optional description of the project.
-         * @throws Throws an error if the project cannot be created or found.
-         * @returns A promise that resolves to an OpenlayerProject object.
+         * Creates a new project in Openlayer or loads an existing one.
+         * @param {string} name - The name of the project.
+         * @param {OpenlayerTaskType} taskType - The type of task associated with the project.
+         * @param {string} [description] - Optional description of the project.
+         * @returns {Promise<OpenlayerProject>} A promise that resolves to an OpenlayerProject object.
+         * @throws {Error} Throws an error if the project cannot be created or found.
          */
         this.createProject = (name, taskType, description) => __awaiter(this, void 0, void 0, function* () {
             try {
@@ -146,10 +149,10 @@ class OpenlayerClient {
         });
         /**
          * Loads an existing inference pipeline from Openlayer based on its name and project ID.
-         * @param name The name of the inference pipeline.
-         * @param projectId The project ID containing the inference pipeline.
-         * @throws Throws an error if the inference pipeline is not found.
-         * @returns A promise that resolves to an OpenlayerInferencePipeline object.
+         * @param {string} projectId - The ID of the project containing the inference pipeline.
+         * @param {string} [name='production'] - The name of the inference pipeline, defaults to 'production'.
+         * @returns {Promise<OpenlayerInferencePipeline>} A promise that resolves to an OpenlayerInferencePipeline object.
+         * @throws {Error} Throws an error if the inference pipeline is not found.
          */
         this.loadInferencePipeline = (projectId, name = 'production') => __awaiter(this, void 0, void 0, function* () {
             const inferencePipelineEndpoint = `/projects/${projectId}/inference-pipelines`;
@@ -174,6 +177,12 @@ class OpenlayerClient {
             }
             return inferencePipeline;
         });
+        /**
+         * Loads an existing project from Openlayer based on its name.
+         * @param {string} name - The name of the project.
+         * @returns {Promise<OpenlayerProject>} A promise that resolves to an OpenlayerProject object.
+         * @throws {Error} Throws an error if the project is not found.
+         */
         this.loadProject = (name) => __awaiter(this, void 0, void 0, function* () {
             const projectsEndpoint = '/projects';
             const projectsQueryParameters = {
@@ -209,21 +218,22 @@ class OpenlayerClient {
 }
 exports.OpenlayerClient = OpenlayerClient;
 class OpenAIMonitor {
+    /**
+     * Constructs an OpenAIMonitor instance.
+     * @param {OpenAIMonitorConstructorProps} props - The configuration properties for the OpenAI and Openlayer clients.
+     */
     constructor({ openAiApiKey, openlayerApiKey, openlayerProjectName, openlayerInferencePipelineName, openlayerServerUrl, }) {
         this.openlayerInferencePipelineName = 'production';
         this.monitoringOn = false;
-        this.formatChatCompletionInput = (messages) => messages
-            .filter(({ role }) => role === 'user')
-            .map(({ content }) => content)
-            .join('\n')
-            .trim();
+        this.formatChatCompletionInput = (messages) => messages.map(({ content, role }, i) => (role === 'user'
+            ? `{{ message_${i} }}`
+            : content));
         /**
-         * Creates a new ChatCompletion instance. If monitoring is not active, an error is thrown.
-         * This function also measures latency and uploads data to Openlayer.
-         * @param body The parameters for creating a chat completion.
-         * @param options Optional request options.
-         * @throws Throws an error if monitoring is not active or if there is no output received from OpenAI.
-         * @returns A promise that resolves to a ChatCompletion or a Stream of ChatCompletionChunks.
+         * Creates a chat completion using the OpenAI client and streams the result to Openlayer.
+         * @param {ChatCompletionCreateParams} body - The parameters for creating a chat completion.
+         * @param {RequestOptions} [options] - Optional request options.
+         * @returns {Promise<ChatCompletion | Stream<ChatCompletionChunk>>} Promise of a ChatCompletion or a Stream
+         * @throws {Error} Throws an error if monitoring is not active or if no output is received from OpenAI.
          */
         this.createChatCompletion = (body, options) => __awaiter(this, void 0, void 0, function* () {
             var _a, e_1, _b, _c;
@@ -238,6 +248,15 @@ class OpenAIMonitor {
             // Accumulate output for streamed responses
             let outputData = '';
             const response = yield this.openAIClient.chat.completions.create(body, options);
+            const prompt = this.formatChatCompletionInput(body.messages);
+            const inputVariableNames = prompt
+                .filter(({ role }) => role === 'user')
+                .map(({ content }) => content);
+            const inputVariables = body.messages
+                .filter(({ role }) => role === 'user')
+                .map(({ content }) => content);
+            const inputVariablesMap = inputVariableNames.reduce((acc, name, i) => (Object.assign(Object.assign({}, acc), { [name]: inputVariables[i] })), {});
+            const config = Object.assign(Object.assign({}, this.openlayerClient.defaultConfig), { inputVariableNames });
             if (body.stream) {
                 const streamedResponse = response;
                 try {
@@ -258,12 +277,7 @@ class OpenAIMonitor {
                 }
                 const endTime = Date.now();
                 const latency = endTime - startTime;
-                this.openlayerClient.streamData({
-                    input: this.formatChatCompletionInput(body.messages),
-                    latency,
-                    output: outputData,
-                    timestamp: startTime,
-                }, inferencePipeline.id);
+                this.openlayerClient.streamData(Object.assign({ latency, output: outputData, prompt, timestamp: startTime }, inputVariablesMap), config, inferencePipeline.id);
             }
             else {
                 const nonStreamedResponse = response;
@@ -274,23 +288,18 @@ class OpenAIMonitor {
                 if (typeof output !== 'string') {
                     throw new Error('No output received from OpenAI.');
                 }
-                this.openlayerClient.streamData({
-                    input: this.formatChatCompletionInput(body.messages),
-                    latency,
+                this.openlayerClient.streamData(Object.assign({ latency,
                     output,
-                    timestamp: startTime,
-                    tokens: (_e = (_d = nonStreamedResponse.usage) === null || _d === void 0 ? void 0 : _d.total_tokens) !== null && _e !== void 0 ? _e : 0,
-                }, inferencePipeline.id);
+                    prompt, timestamp: startTime, tokens: (_e = (_d = nonStreamedResponse.usage) === null || _d === void 0 ? void 0 : _d.total_tokens) !== null && _e !== void 0 ? _e : 0 }, inputVariablesMap), config, inferencePipeline.id);
             }
             return response;
         });
         /**
-         * Creates a new Completion instance. If monitoring is not active, an error is thrown.
-         * This function also measures latency and uploads data to Openlayer.
-         * @param body The parameters for creating a completion.
-         * @param options Optional request options.
-         * @throws Throws an error if monitoring is not active or if no prompt is provided.
-         * @returns A promise that resolves to a Completion or a Stream of Completions.
+         * Creates a completion using the OpenAI client and streams the result to Openlayer.
+         * @param {CompletionCreateParams} body - The parameters for creating a completion.
+         * @param {RequestOptions} [options] - Optional request options.
+         * @returns {Promise<Completion | Stream<Completion>>} Promise that resolves to a Completion or a Stream.
+         * @throws {Error} Throws an error if monitoring is not active or if no prompt is provided.
          */
         this.createCompletion = (body, options) => __awaiter(this, void 0, void 0, function* () {
             var _g, e_2, _h, _j;
@@ -309,6 +318,7 @@ class OpenAIMonitor {
             let outputData = '';
             let tokensData = 0;
             const response = yield this.openAIClient.completions.create(body, options);
+            const config = Object.assign(Object.assign({}, this.openlayerClient.defaultConfig), { inputVariableNames: ['input'] });
             if (body.stream) {
                 const streamedResponse = response;
                 try {
@@ -336,7 +346,7 @@ class OpenAIMonitor {
                     output: outputData,
                     timestamp: startTime,
                     tokens: tokensData,
-                }, inferencePipeline.id);
+                }, config, inferencePipeline.id);
             }
             else {
                 const nonStreamedResponse = response;
@@ -349,7 +359,7 @@ class OpenAIMonitor {
                     output: nonStreamedResponse.choices[0].text,
                     timestamp: startTime,
                     tokens: (_o = (_m = nonStreamedResponse.usage) === null || _m === void 0 ? void 0 : _m.total_tokens) !== null && _o !== void 0 ? _o : 0,
-                }, inferencePipeline.id);
+                }, config, inferencePipeline.id);
             }
             return response;
         });

package/examples/langchain.mjs

@@ -0,0 +1,45 @@
+/*
+ * This example shows how to use Openlayer to monitor your LangChain workflows.
+ */
+
+import { ChatOpenAI } from 'langchain/chat_models/openai';
+import { OpenlayerClient } from 'openlayer';
+
+// Instantiate the Openlayer client with your API key
+const openlayer = new OpenlayerClient({
+  openlayerApiKey: 'YOUR_OPENLAYER_API_KEY',
+});
+
+// Create or load your project
+const project = await openlayer.createProject('YOUR_PROJECT_NAME', 'llm-base');
+
+/*
+ * Create or load an inference pipeline from your project.
+ * If no name is specified, it will default to 'production'
+ */
+const inferencePipeline = await openlayer.createInferencePipeline(project.id);
+const chatModel = new ChatOpenAI();
+const inputs = [
+  'What is the meaning of life?',
+  'What would be a good name for a company that makes colorful socks?',
+];
+
+await Promise.all(
+  inputs.map(async (input) => {
+    // Call the LLM
+    const output = await chatModel.predict(input);
+
+    // Stream the results to Openlayer
+    await openlayer.streamData(
+      {
+        input,
+        output,
+      },
+      {
+        ...openlayer.defaultConfig,
+        inputVariableNames: ['input'],
+      },
+      inferencePipeline.id
+    );
+  })
+);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "openlayer",
-  "version": "0.1.11",
+  "version": "0.1.13",
   "description": "The Openlayer TypeScript client",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",