langsmith 0.0.1-0

package/README.md

@@ -0,0 +1,262 @@
+# LangSmith Client SDK
+
+This package contains the TypeScript client for interacting with the [LangSmith platform](https://www.langchain.plus/).
+
+To install:
+
+```bash
+yarn add langsmith
+```
+
+LangSmith helps you and your team develop and evaluate language models and intelligent agents. It is compatible with any LLM Application and provides seamless integration with [LangChain](https://github.com/hwchase17/langchainjs), a widely recognized open-source framework that simplifies the process for developers to create powerful language model applications.
+
+> **Note**: You can enjoy the benefits of LangSmith without using the LangChain open-source packages! To get started with your own proprietary framework, set up your account and then skip to [Logging Traces Outside LangChain](#logging-traces-outside-langchain).
+
+A typical workflow looks like:
+
+1. Set up an account with LangSmith or host your [local server](https://docs.langchain.plus/docs/getting-started/local_installation).
+2. Log traces.
+3. Debug, Create Datasets, and Evaluate Runs.
+
+We'll walk through these steps in more detail below.
+
+## 1. Connect to LangSmith
+
+Sign up for [LangSmith](https://www.langchain.plus/) using your GitHub, Discord accounts, or an email address and password. If you sign up with an email, make sure to verify your email address before logging in.
+
+Then, create a unique API key on the [Settings Page](https://www.langchain.plus/settings), which is found in the menu at the top right corner of the page.
+
+Note: Save the API Key in a secure location. It will not be shown again.
+
+## 2. Log Traces
+
+You can log traces natively in your LangChain application or using a LangSmith RunTree.
+
+### Logging Traces with LangChain
+
+LangSmith seamlessly integrates with the JavaScript LangChain library to record traces from your LLM applications.
+
+```bash
+yarn add langchain
+```
+
+1. **Copy the environment variables from the Settings Page and add them to your application.**
+
+Tracing can be activated by setting the following environment variables or by manually specifying the LangChainTracer.
+
+```typescript
+process.env["LANGCHAIN_TRACING_V2"] = "true";
+process.env["LANGCHAIN_ENDPOINT"] = "https://api.langchain.plus"; // or your own server
+process.env["LANGCHAIN_API_KEY"] = "<YOUR-LANGCHAINPLUS-API-KEY>";
+// process.env["LANGCHAIN_PROJECT"] = "My Project Name"; // Optional: "default" is used if not set
+```
+
+> **Tip:** Projects are groups of traces. All runs are logged to a project. If not specified, the project is set to `default`.
+
+2. **Run an Agent, Chain, or Language Model in LangChain**
+
+If the environment variables are correctly set, your application will automatically connect to the LangSmith platform.
+
+```typescript
+import { ChatOpenAI } from "langchain/chat_models/openai";
+
+const chat = new ChatOpenAI({ temperature: 0 });
+const response = await chat.predict(
+  "Translate this sentence from English to French. I love programming."
+);
+console.log(response);
+```
+
+### Logging Traces Outside LangChain
+
+_Note: this API is experimental and may change in the future_
+
+You can still use the LangSmith development platform without depending on any
+LangChain code. You can connect either by setting the appropriate environment variables,
+or by directly specifying the connection information in the RunTree.
+
+1. **Copy the environment variables from the Settings Page and add them to your application.**
+
+```typescript
+process.env["LANGCHAIN_ENDPOINT"] = "https://api.langchain.plus"; // or your own server
+process.env["LANGCHAIN_API_KEY"] = "<YOUR-LANGCHAINPLUS-API-KEY>";
+// process.env["LANGCHAIN_PROJECT"] = "My Project Name"; // Optional: "default" is used if not set
+```
+
+2. **Log traces using a RunTree.**
+
+A RunTree tracks your application. Each RunTree object is required to have a name and run_type. These and other important attributes are as follows:
+
+- `name`: `string` - used to identify the component's purpose
+- `run_type`: `string` - Currently one of "llm", "chain" or "tool"; more options will be added in the future
+- `inputs`: `Record<string, any>` - the inputs to the component
+- `outputs`: `Optional<Record<string, any>>` - the (optional) returned values from the component
+- `error`: `Optional<string>` - Any error messages that may have arisen during the call
+
+```typescript
+import { RunTree, RunTreeConfig } from "langsmith";
+
+const parentRunConfig: RunTreeConfig = {
+  name: "My Chat Bot",
+  run_type: "chain",
+  inputs: {
+    text: "Summarize this morning's meetings.",
+  },
+  serialized: {}, // Serialized representation of this chain
+  // project_name: "Defaults to the LANGCHAIN_PROJECT env var"
+  // apiUrl: "Defaults to the LANGCHAIN_ENDPOINT env var"
+  // apiKey: "Defaults to the LANGCHAIN_API_KEY env var"
+};
+
+const parentRun = new RunTree(parentRunConfig);
+
+const childLlmRun = await parentRun.createChild({
+  name: "My Proprietary LLM",
+  run_type: "llm",
+  inputs: {
+    prompts: [
+      "You are an AI Assistant. The time is XYZ." +
+        " Summarize this morning's meetings.",
+    ],
+  },
+});
+
+await childLlmRun.end({
+  outputs: {
+    generations: [
+      "I should use the transcript_loader tool" +
+        " to fetch meeting_transcripts from XYZ",
+    ],
+  },
+});
+
+const childToolRun = await parentRun.createChild({
+  name: "transcript_loader",
+  run_type: "tool",
+  inputs: {
+    date: "XYZ",
+    content_type: "meeting_transcripts",
+  },
+});
+
+await childToolRun.end({
+  outputs: {
+    meetings: ["Meeting1 notes.."],
+  },
+});
+
+const childChainRun = await parentRun.createChild({
+  name: "Unreliable Component",
+  run_type: "tool",
+  inputs: {
+    input: "Summarize these notes...",
+  },
+});
+
+try {
+  // .... the component does work
+  throw new Error("Something went wrong");
+} catch (e) {
+  await childChainRun.end({
+    error: `I errored again ${e.message}`,
+  });
+}
+
+await parentRun.end({
+  outputs: {
+    output: ["The meeting notes are as follows:..."],
+  },
+});
+
+await parentRun.postRun({
+  exclude_child_runs: false,
+});
+```
+
+### Create a Dataset from Existing Runs
+
+Once your runs are stored in LangSmith, you can convert them into a dataset.
+For this example, we will do so using the Client, but you can also do this using
+the web interface, as explained in the [LangSmith docs](https://docs.langchain.plus/docs/).
+
+```typescript
+import { Client } from "langsmith/client";
+const client = new Client({
+  // apiUrl: "https://api.langchain.com", // Defaults to the LANGCHAIN_ENDPOINT env var
+  // apiKey: "my_api_key", // Defaults to the LANGCHAIN_API_KEY env var
+  /* callerOptions: {
+         maxConcurrency?: Infinity; // Maximum number of concurrent requests to make
+         maxRetries?: 6; // Maximum number of retries to make
+    */
+});
+const datasetName = "Example Dataset";
+// We will only use examples from the top level AgentExecutor run here,
+// and exclude runs that errored.
+const runs = await client.listRuns({
+  projectName: "my_project",
+  executionOrder: 1,
+  error: false,
+});
+
+const dataset = await client.createDataset(datasetName, {
+  description: "An example dataset",
+});
+
+for (const run of runs) {
+  await client.createExample(run.inputs, run.outputs ?? {}, {
+    datasetId: dataset.id,
+  });
+}
+```
+
+# Evaluating Runs
+
+You can run evaluations directly using the LangSmith client.
+
+```ts
+import { StringEvaluator } from "langsmith/evaluation";
+
+function jaccardChars(output: string, answer: string): number {
+  const predictionChars = new Set(output.trim().toLowerCase());
+  const answerChars = new Set(answer.trim().toLowerCase());
+  const intersection = [...predictionChars].filter((x) => answerChars.has(x));
+  const union = new Set([...predictionChars, ...answerChars]);
+  return intersection.length / union.size;
+}
+
+async function grader(config: {
+  input: string;
+  prediction: string;
+  answer?: string;
+}): Promise<{ score: number; value: string }> {
+  let value: string;
+  let score: number;
+  if (config.answer === null || config.answer === undefined) {
+    value = "AMBIGUOUS";
+    score = 0.5;
+  } else {
+    score = jaccardChars(config.prediction, config.answer);
+    value = score > 0.9 ? "CORRECT" : "INCORRECT";
+  }
+  return { score: score, value: value };
+}
+
+const evaluator = new StringEvaluator({
+  evaluationName: "Jaccard",
+  gradingFunction: grader,
+});
+
+const runs = await client.listRuns({
+  projectName: "my_project",
+  executionOrder: 1,
+  error: false,
+});
+
+for (const run of runs) {
+  client.evaluateRun(run, evaluator);
+}
+```
+
+## Additional Documentation
+
+To learn more about the LangSmith platform, check out the [docs](https://docs.langchain.plus/docs/).

package/client.cjs

@@ -0,0 +1 @@
+module.exports = require('./dist/client.cjs');

package/client.d.ts

@@ -0,0 +1 @@
+export * from './dist/client.js'

package/client.js

@@ -0,0 +1 @@
+export * from './dist/client.js'

package/dist/cli/docker-compose.ngrok.yaml

@@ -0,0 +1,17 @@
+version: '3'
+services:
+  ngrok:
+    image: ngrok/ngrok:latest
+    restart: unless-stopped
+    command:
+      - "start"
+      - "--all"
+      - "--config"
+      - "/etc/ngrok.yml"
+    volumes:
+      - ./ngrok_config.yaml:/etc/ngrok.yml
+    ports:
+      - 4040:4040
+  langchain-backend:
+    depends_on:
+      - ngrok

package/dist/cli/docker-compose.yaml

@@ -0,0 +1,43 @@
+version: "3"
+services:
+  langchain-frontend:
+    image: langchain/${_LANGCHAINPLUS_IMAGE_PREFIX-}langchainplus-frontend:latest
+    ports:
+      - 80:80
+    environment:
+      - REACT_APP_BACKEND_URL=http://localhost:1984
+    depends_on:
+      - langchain-backend
+    volumes:
+      - ./conf/nginx.conf:/etc/nginx/default.conf:ro
+  langchain-backend:
+    image: langchain/${_LANGCHAINPLUS_IMAGE_PREFIX-}langchainplus-backend:latest
+    environment:
+      - PORT=1984
+      - LANGCHAIN_ENV=local_docker
+      - LOG_LEVEL=warning
+      - OPENAI_API_KEY=${OPENAI_API_KEY}
+    ports:
+      - 1984:1984
+    depends_on:
+      - langchain-db
+  langchain-db:
+    image: postgres:14.1
+    command:
+      [
+        "postgres",
+        "-c",
+        "log_min_messages=WARNING",
+        "-c",
+        "client_min_messages=WARNING",
+      ]
+    environment:
+      - POSTGRES_PASSWORD=postgres
+      - POSTGRES_USER=postgres
+      - POSTGRES_DB=postgres
+    volumes:
+      - langchain-db-data:/var/lib/postgresql/data
+    ports:
+      - 5433:5432
+volumes:
+  langchain-db-data:

package/dist/cli/main.cjs

@@ -0,0 +1,278 @@
+#!/usr/bin/env node
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const util = __importStar(require("util"));
+const commander_1 = require("commander");
+const child_process = __importStar(require("child_process"));
+const env_js_1 = require("../utils/env.cjs");
+const currentFileName = __filename;
+const currentDirName = __dirname;
+const exec = util.promisify(child_process.exec);
+const program = new commander_1.Command();
+async function getDockerComposeCommand() {
+    try {
+        await exec("docker compose --version");
+        return ["docker", "compose"];
+    }
+    catch {
+        try {
+            await exec("docker-compose --version");
+            return ["docker-compose"];
+        }
+        catch {
+            throw new Error("Neither 'docker compose' nor 'docker-compose' commands are available. Please install the Docker server following the instructions for your operating system at https://docs.docker.com/engine/install/");
+        }
+    }
+}
+async function pprintServices(servicesStatus) {
+    const services = [];
+    for (const service of servicesStatus) {
+        const serviceStatus = {
+            Service: String(service["Service"]),
+            Status: String(service["Status"]),
+        };
+        const publishers = service["Publishers"] || [];
+        if (publishers) {
+            serviceStatus["PublishedPorts"] = publishers
+                .map((publisher) => String(publisher["PublishedPort"]))
+                .join(", ");
+        }
+        services.push(serviceStatus);
+    }
+    const maxServiceLen = Math.max(...services.map((service) => service["Service"].length));
+    const maxStateLen = Math.max(...services.map((service) => service["Status"].length));
+    const serviceMessage = [
+        "\n" +
+            "Service".padEnd(maxServiceLen + 2) +
+            "Status".padEnd(maxStateLen + 2) +
+            "Published Ports",
+    ];
+    for (const service of services) {
+        const serviceStr = service["Service"].padEnd(maxServiceLen + 2);
+        const stateStr = service["Status"].padEnd(maxStateLen + 2);
+        const portsStr = service["PublishedPorts"] || "";
+        serviceMessage.push(serviceStr + stateStr + portsStr);
+    }
+    let langchainEndpoint = "http://localhost:1984";
+    const usedNgrok = services.some((service) => service["Service"].includes("ngrok"));
+    if (usedNgrok) {
+        langchainEndpoint = await getNgrokUrl();
+    }
+    serviceMessage.push("\nTo connect, set the following environment variables" +
+        " in your LangChain application:" +
+        "\nLANGCHAIN_TRACING_V2=true" +
+        `\nLANGCHAIN_ENDPOINT=${langchainEndpoint}`);
+    console.info(serviceMessage.join("\n"));
+}
+async function getNgrokUrl() {
+    const ngrokUrl = "http://localhost:4040/api/tunnels";
+    try {
+        // const response = await axios.get(ngrokUrl);
+        const response = await fetch(ngrokUrl);
+        if (response.status !== 200) {
+            throw new Error(`Could not connect to ngrok console. ${response.status}, ${response.statusText}`);
+        }
+        const result = await response.json();
+        const exposedUrl = result["tunnels"][0]["public_url"];
+        return exposedUrl;
+    }
+    catch (error) {
+        throw new Error(`Could not connect to ngrok console. ${error}`);
+    }
+}
+async function createNgrokConfig(authToken) {
+    const configPath = path.join(currentDirName, "ngrok_config.yaml");
+    // Check if is a directory
+    if (fs.existsSync(configPath) && fs.lstatSync(configPath).isDirectory()) {
+        fs.rmdirSync(configPath, { recursive: true });
+    }
+    else if (fs.existsSync(configPath)) {
+        fs.unlinkSync(configPath);
+    }
+    let ngrokConfig = `
+region: us
+tunnels:
+  langchain:
+    addr: langchain-backend:1984
+    proto: http
+version: '2'
+`;
+    if (authToken !== null) {
+        ngrokConfig += `authtoken: ${authToken}`;
+    }
+    fs.writeFileSync(configPath, ngrokConfig);
+    return configPath;
+}
+class PlusCommand {
+    constructor({ dockerComposeCommand }) {
+        Object.defineProperty(this, "dockerComposeCommand", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: []
+        });
+        Object.defineProperty(this, "dockerComposeFile", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: ""
+        });
+        Object.defineProperty(this, "ngrokPath", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: ""
+        });
+        this.dockerComposeCommand = dockerComposeCommand;
+        this.dockerComposeFile = path.join(path.dirname(currentFileName), "docker-compose.yaml");
+        this.ngrokPath = path.join(path.dirname(currentFileName), "docker-compose.ngrok.yaml");
+    }
+    static async create() {
+        const dockerComposeCommand = await getDockerComposeCommand();
+        return new PlusCommand({ dockerComposeCommand });
+    }
+    async start(args) {
+        if (args.dev) {
+            (0, env_js_1.setEnvironmentVariable)("_LANGCHAINPLUS_IMAGE_PREFIX", "rc-");
+        }
+        if (args.openaiApiKey) {
+            (0, env_js_1.setEnvironmentVariable)("OPENAI_API_KEY", args.openaiApiKey);
+        }
+        if (args.expose) {
+            await this.startAndExpose(args.ngrokAuthtoken);
+        }
+        else {
+            await this.startLocal();
+        }
+    }
+    async pull(args) {
+        if (args.dev) {
+            (0, env_js_1.setEnvironmentVariable)("_LANGCHAINPLUS_IMAGE_PREFIX", "rc-");
+        }
+        const command = [
+            ...this.dockerComposeCommand,
+            "-f",
+            this.dockerComposeFile,
+            "pull",
+        ];
+        await exec(command.join(" "));
+    }
+    async startLocal() {
+        const command = [
+            ...this.dockerComposeCommand,
+            "-f",
+            this.dockerComposeFile,
+            "up",
+            "--pull=always",
+            "--quiet-pull",
+            "--wait",
+        ];
+        await exec(command.join(" "));
+        console.info("LangSmith server is running at http://localhost:1984.  To connect locally, set the following environment variable when running your LangChain application.");
+        console.info("\tLANGCHAIN_TRACING_V2=true");
+    }
+    async startAndExpose(ngrokAuthToken) {
+        const configPath = await createNgrokConfig(ngrokAuthToken);
+        const command = [
+            ...this.dockerComposeCommand,
+            "-f",
+            this.dockerComposeFile,
+            "-f",
+            this.ngrokPath,
+            "up",
+            "--pull=always",
+            "--quiet-pull",
+            "--wait",
+        ];
+        await exec(command.join(" "));
+        console.info("ngrok is running. You can view the dashboard at http://0.0.0.0:4040");
+        const ngrokUrl = await getNgrokUrl();
+        console.info("LangSmith server is running at http://localhost:1984. To connect remotely, set the following environment variable when running your LangChain application.");
+        console.info("\tLANGCHAIN_TRACING_V2=true");
+        console.info(`\tLANGCHAIN_ENDPOINT=${ngrokUrl}`);
+        fs.unlinkSync(configPath);
+    }
+    async stop() {
+        const command = [
+            ...this.dockerComposeCommand,
+            "-f",
+            this.dockerComposeFile,
+            "-f",
+            this.ngrokPath,
+            "down",
+        ];
+        await exec(command.join(" "));
+    }
+    async status() {
+        const command = [
+            ...this.dockerComposeCommand,
+            "-f",
+            this.dockerComposeFile,
+            "ps",
+            "--format",
+            "json",
+        ];
+        const result = await exec(command.join(" "));
+        const servicesStatus = JSON.parse(result.stdout);
+        if (servicesStatus) {
+            console.info("The LangSmith server is currently running.");
+            await pprintServices(servicesStatus);
+        }
+        else {
+            console.info("The LangSmith server is not running.");
+        }
+    }
+}
+const startCommand = new commander_1.Command("start")
+    .description("Start the LangSmith server")
+    .option("--expose", "Expose the server to the internet via ngrok (requires ngrok to be installed)")
+    .option("--ngrok-authtoken <ngrokAuthtoken>", "Your ngrok auth token. If this is set, --expose is implied.")
+    .option("--dev", "Run the development version of the LangSmith server")
+    .option("--openai-api-key <openaiApiKey>", "Your OpenAI API key. If this is set, the server will be able to process text and return enhanced plus results.")
+    .action(async (args) => (await PlusCommand.create()).start(args));
+const stopCommand = new commander_1.Command("stop")
+    .command("stop")
+    .description("Stop the LangSmith server")
+    .action(async () => (await PlusCommand.create()).stop());
+const pullCommand = new commander_1.Command("pull")
+    .command("pull")
+    .description("Pull the latest version of the LangSmith server")
+    .option("--dev", "Pull the development version of the LangSmith server")
+    .action(async (args) => (await PlusCommand.create()).pull(args));
+const statusCommand = new commander_1.Command("status")
+    .command("status")
+    .description("Get the status of the LangSmith server")
+    .action(async () => (await PlusCommand.create()).status());
+program
+    .command("plus")
+    .description("Manage the LangSmith server")
+    .addCommand(startCommand)
+    .addCommand(stopCommand)
+    .addCommand(pullCommand)
+    .addCommand(statusCommand);
+program.parse(process.argv);

package/dist/cli/main.d.ts

@@ -0,0 +1 @@
+export {};