openlayer 0.1.12 → 0.1.14

package/README.md

@@ -0,0 +1,35 @@
+<div align="left">
+  <img src="static/logo.png"><br>
+</div>
+
+# Openlayer | JavaScript/TypeScript Library
+
+[![npm version](https://badge.fury.io/js/openlayer.svg)](https://badge.fury.io/js/openlayer)
+
+## What is it?
+
+Openlayer is a debugging workspace for ML & Data Science. Openlayer combines and builds upon SOTA techniques in explainability, model and dataset versioning, synthetic data generation, data-centric testing and much more to form a powerful, **unified platform for model development**.
+
+👉 [Join our Discord community!](https://discord.gg/t6wS2g6MMB) We'd love to meet you and help you get started with Openlayer!
+
+This is the official JavaScript/TypeScript library for interacting with the Openlayer platform. Navigate [here](https://docs.openlayer.com) for a quickstart guide and for in-depth tutorials.
+
+## Main Features
+
+This library's primary function is to enable you to easily package your models and datasets and add them to your Openlayer account.
+
+## Installation
+
+Install with npm
+
+```console
+npm i openlayer
+```
+
+## Documentation
+
+The official documentation for this library can be found [here](https://docs.openlayer.com).
+
+## Contributing
+
+All contributions, bug reports, bug fixes, documentation improvements, enhancements, and ideas are welcome! Just send us a message on [Discord](https://discord.gg/t6wS2g6MMB).

package/dist/index.d.ts

@@ -1,16 +1,12 @@
 import { RequestOptions } from 'openai/core';
-import { ChatCompletion, ChatCompletionChunk, ChatCompletionCreateParams, Completion, CompletionCreateParams } from 'openai/resources';
+import { ChatCompletion, ChatCompletionChunk, ChatCompletionCreateParams, ChatCompletionMessageParam, Completion, CompletionCreateParams } from 'openai/resources';
 import { Stream } from 'openai/streaming';
 /**
  * Represents the data structure for a chat completion.
+ * Object keys represent a column name and the values represent the column value.
  */
-export interface ChatCompletionData {
-    [key: string]: any;
-    /**
-     * The input data for the chat completion. It can be a string, an array of strings,
-     * or a nested array of numbers.
-     */
-    input: string | string[] | number[] | number[][];
+export interface StreamingData {
+    [columnName: string]: any;
     /**
      * The latency of the chat completion in milliseconds. Optional.
      */
@@ -28,6 +24,43 @@ export interface ChatCompletionData {
      */
     tokens?: number;
 }
+/**
+ * Configuration settings for uploading chat completion data to Openlayer.
+ */
+interface StreamingDataConfig {
+    /**
+     * The name of the column that stores the ground truth data. Can be null.
+     */
+    groundTruthColumnName: string | null;
+    /**
+     * The name of the column that stores inference IDs. Can be null.
+     */
+    inferenceIdColumnName: string | null;
+    /**
+     * An array of names for input variable columns. Can be null.
+     */
+    inputVariableNames?: string[] | null;
+    /**
+     * The name of the column that stores latency data. Can be null.
+     */
+    latencyColumnName: string | null;
+    /**
+     * The name of the column that stores the number of tokens. Can be null.
+     */
+    numOfTokenColumnName: string | null;
+    /**
+     * The name of the column that stores output data. Can be null.
+     */
+    outputColumnName: string | null;
+    /**
+     * The full prompt history for the chat completion.
+     */
+    prompt?: ChatCompletionMessageParam[];
+    /**
+     * The name of the column that stores timestamp data. Can be null.
+     */
+    timestampColumnName: string | null;
+}
 type OpenlayerClientConstructorProps = {
     openlayerApiKey?: string;
     openlayerInferencePipelineName?: string;
@@ -96,7 +129,7 @@ type OpenlayerSampleVolumeGraph = {
 type OpenlayerTaskType = 'llm-base' | 'tabular-classification' | 'tabular-regression' | 'text-classification';
 export declare class OpenlayerClient {
     private openlayerApiKey?;
-    private openlayerDefaultDataConfig;
+    defaultConfig: StreamingDataConfig;
     private openlayerServerUrl;
     private version;
     /**
@@ -108,12 +141,12 @@ export declare class OpenlayerClient {
     private resolvedQuery;
     /**
      * Streams data to the Openlayer inference pipeline.
-     * @param {ChatCompletionData} data - The chat completion data to be streamed.
+     * @param {StreamingData} data - The chat completion data to be streamed.
      * @param {string} inferencePipelineId - The ID of the Openlayer inference pipeline to which data is streamed.
      * @returns {Promise<void>} A promise that resolves when the data has been successfully streamed.
      * @throws {Error} Throws an error if the Openlayer API key is not set or an error occurs in the streaming process.
      */
-    streamData: (data: ChatCompletionData, inferencePipelineId: string) => Promise<void>;
+    streamData: (data: StreamingData, config: StreamingDataConfig, inferencePipelineId: string) => Promise<void>;
     /**
      * Creates a new inference pipeline in Openlayer or loads an existing one.
      * @param {string} projectId - The ID of the project containing the inference pipeline.

package/dist/index.js

@@ -27,10 +27,9 @@ class OpenlayerClient {
      * @throws {Error} Throws an error if the Openlayer API key is not provided.
      */
     constructor({ openlayerApiKey, openlayerServerUrl, }) {
-        this.openlayerDefaultDataConfig = {
+        this.defaultConfig = {
             groundTruthColumnName: null,
             inferenceIdColumnName: 'id',
-            inputVariableNames: ['input'],
             latencyColumnName: 'latency',
             numOfTokenColumnName: 'tokens',
             outputColumnName: 'output',
@@ -41,12 +40,12 @@ class OpenlayerClient {
         this.resolvedQuery = (endpoint, args = {}) => (0, request_1.resolvedQuery)(this.openlayerServerUrl, endpoint, args);
         /**
          * Streams data to the Openlayer inference pipeline.
-         * @param {ChatCompletionData} data - The chat completion data to be streamed.
+         * @param {StreamingData} data - The chat completion data to be streamed.
          * @param {string} inferencePipelineId - The ID of the Openlayer inference pipeline to which data is streamed.
          * @returns {Promise<void>} A promise that resolves when the data has been successfully streamed.
          * @throws {Error} Throws an error if the Openlayer API key is not set or an error occurs in the streaming process.
          */
-        this.streamData = (data, inferencePipelineId) => __awaiter(this, void 0, void 0, function* () {
+        this.streamData = (data, config, inferencePipelineId) => __awaiter(this, void 0, void 0, function* () {
             var _a;
             if (!this.openlayerApiKey) {
                 throw new Error('Openlayer API key are required for streaming data.');
@@ -56,7 +55,7 @@ class OpenlayerClient {
                 const dataStreamQuery = this.resolvedQuery(dataStreamEndpoint);
                 const response = yield fetch(dataStreamQuery, {
                     body: JSON.stringify({
-                        config: this.openlayerDefaultDataConfig,
+                        config,
                         rows: [
                             Object.assign(Object.assign({}, data), { id: (0, uuid_1.v4)(), timestamp: Math.round(((_a = data.timestamp) !== null && _a !== void 0 ? _a : Date.now()) / 1000) }),
                         ],
@@ -125,7 +124,7 @@ class OpenlayerClient {
             catch (_c) {
                 const projectsEndpoint = '/projects';
                 const projectsQuery = this.resolvedQuery(projectsEndpoint);
-                const projectsResponse = yield fetch(projectsQuery, {
+                const response = yield fetch(projectsQuery, {
                     body: JSON.stringify({
                         description,
                         name,
@@ -137,9 +136,10 @@ class OpenlayerClient {
                     },
                     method: 'POST',
                 });
-                const { items: projects } = yield projectsResponse.json();
+                const data = yield response.json();
+                const { items: projects, error } = data;
                 if (!Array.isArray(projects)) {
-                    throw new Error('Invalid response from Openlayer');
+                    throw new Error(typeof error === 'string' ? error : 'Invalid response from Openlayer');
                 }
                 const project = projects.find((p) => p.name === name);
                 if (!(project === null || project === void 0 ? void 0 : project.id)) {
@@ -191,16 +191,17 @@ class OpenlayerClient {
                 version: this.version,
             };
             const projectsQuery = this.resolvedQuery(projectsEndpoint, projectsQueryParameters);
-            const projectsResponse = yield fetch(projectsQuery, {
+            const response = yield fetch(projectsQuery, {
                 headers: {
                     Authorization: `Bearer ${this.openlayerApiKey}`,
                     'Content-Type': 'application/json',
                 },
                 method: 'GET',
             });
-            const { items: projects } = yield projectsResponse.json();
+            const data = yield response.json();
+            const { items: projects, error } = data;
             if (!Array.isArray(projects)) {
-                throw new Error('Invalid response from Openlayer');
+                throw new Error(typeof error === 'string' ? error : 'Invalid response from Openlayer');
             }
             const project = projects.find((p) => p.name === name);
             if (!(project === null || project === void 0 ? void 0 : project.id)) {
@@ -226,11 +227,9 @@ class OpenAIMonitor {
     constructor({ openAiApiKey, openlayerApiKey, openlayerProjectName, openlayerInferencePipelineName, openlayerServerUrl, }) {
         this.openlayerInferencePipelineName = 'production';
         this.monitoringOn = false;
-        this.formatChatCompletionInput = (messages) => messages
-            .filter(({ role }) => role === 'user')
-            .map(({ content }) => content)
-            .join('\n')
-            .trim();
+        this.formatChatCompletionInput = (messages) => messages.map(({ content, role }, i) => (role === 'user'
+            ? `{{ message_${i} }}`
+            : content));
         /**
          * Creates a chat completion using the OpenAI client and streams the result to Openlayer.
          * @param {ChatCompletionCreateParams} body - The parameters for creating a chat completion.
@@ -251,6 +250,15 @@ class OpenAIMonitor {
             // Accumulate output for streamed responses
             let outputData = '';
             const response = yield this.openAIClient.chat.completions.create(body, options);
+            const prompt = this.formatChatCompletionInput(body.messages);
+            const inputVariableNames = prompt
+                .filter(({ role }) => role === 'user')
+                .map(({ content }) => content);
+            const inputVariables = body.messages
+                .filter(({ role }) => role === 'user')
+                .map(({ content }) => content);
+            const inputVariablesMap = inputVariableNames.reduce((acc, name, i) => (Object.assign(Object.assign({}, acc), { [name]: inputVariables[i] })), {});
+            const config = Object.assign(Object.assign({}, this.openlayerClient.defaultConfig), { inputVariableNames });
             if (body.stream) {
                 const streamedResponse = response;
                 try {
@@ -271,12 +279,7 @@ class OpenAIMonitor {
                 }
                 const endTime = Date.now();
                 const latency = endTime - startTime;
-                this.openlayerClient.streamData({
-                    input: this.formatChatCompletionInput(body.messages),
-                    latency,
-                    output: outputData,
-                    timestamp: startTime,
-                }, inferencePipeline.id);
+                this.openlayerClient.streamData(Object.assign({ latency, output: outputData, prompt, timestamp: startTime }, inputVariablesMap), config, inferencePipeline.id);
             }
             else {
                 const nonStreamedResponse = response;
@@ -287,13 +290,9 @@ class OpenAIMonitor {
                 if (typeof output !== 'string') {
                     throw new Error('No output received from OpenAI.');
                 }
-                this.openlayerClient.streamData({
-                    input: this.formatChatCompletionInput(body.messages),
-                    latency,
+                this.openlayerClient.streamData(Object.assign({ latency,
                     output,
-                    timestamp: startTime,
-                    tokens: (_e = (_d = nonStreamedResponse.usage) === null || _d === void 0 ? void 0 : _d.total_tokens) !== null && _e !== void 0 ? _e : 0,
-                }, inferencePipeline.id);
+                    prompt, timestamp: startTime, tokens: (_e = (_d = nonStreamedResponse.usage) === null || _d === void 0 ? void 0 : _d.total_tokens) !== null && _e !== void 0 ? _e : 0 }, inputVariablesMap), config, inferencePipeline.id);
             }
             return response;
         });
@@ -321,6 +320,7 @@ class OpenAIMonitor {
             let outputData = '';
             let tokensData = 0;
             const response = yield this.openAIClient.completions.create(body, options);
+            const config = Object.assign(Object.assign({}, this.openlayerClient.defaultConfig), { inputVariableNames: ['input'] });
             if (body.stream) {
                 const streamedResponse = response;
                 try {
@@ -348,7 +348,7 @@ class OpenAIMonitor {
                     output: outputData,
                     timestamp: startTime,
                     tokens: tokensData,
-                }, inferencePipeline.id);
+                }, config, inferencePipeline.id);
             }
             else {
                 const nonStreamedResponse = response;
@@ -361,7 +361,7 @@ class OpenAIMonitor {
                     output: nonStreamedResponse.choices[0].text,
                     timestamp: startTime,
                     tokens: (_o = (_m = nonStreamedResponse.usage) === null || _m === void 0 ? void 0 : _m.total_tokens) !== null && _o !== void 0 ? _o : 0,
-                }, inferencePipeline.id);
+                }, config, inferencePipeline.id);
             }
             return response;
         });

package/examples/langchain.mjs

@@ -0,0 +1,45 @@
+/*
+ * This example shows how to use Openlayer to monitor your LangChain workflows.
+ */
+
+import { ChatOpenAI } from 'langchain/chat_models/openai';
+import { OpenlayerClient } from 'openlayer';
+
+// Instantiate the Openlayer client with your API key
+const openlayer = new OpenlayerClient({
+  openlayerApiKey: 'YOUR_OPENLAYER_API_KEY',
+});
+
+// Create or load your project
+const project = await openlayer.createProject('YOUR_PROJECT_NAME', 'llm-base');
+
+/*
+ * Create or load an inference pipeline from your project.
+ * If no name is specified, it will default to 'production'
+ */
+const inferencePipeline = await openlayer.createInferencePipeline(project.id);
+const chatModel = new ChatOpenAI();
+const inputs = [
+  'What is the meaning of life?',
+  'What would be a good name for a company that makes colorful socks?',
+];
+
+await Promise.all(
+  inputs.map(async (input) => {
+    // Call the LLM
+    const output = await chatModel.predict(input);
+
+    // Stream the results to Openlayer
+    await openlayer.streamData(
+      {
+        input,
+        output,
+      },
+      {
+        ...openlayer.defaultConfig,
+        inputVariableNames: ['input'],
+      },
+      inferencePipeline.id
+    );
+  })
+);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "openlayer",
-  "version": "0.1.12",
+  "version": "0.1.14",
   "description": "The Openlayer TypeScript client",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",