@rawdash/connector-langsmith 0.1.0

package/dist/index.js

@@ -0,0 +1,613 @@
+// ../../connector-shared/dist/index.js
+var HTTP_CLIENT_VERSION = "0.0.0";
+var DEFAULT_USER_AGENT = `rawdash-connector/${HTTP_CLIENT_VERSION} (+https://rawdash.dev)`;
+function connectorUserAgent(connectorId) {
+  return `rawdash-connector-${connectorId}/${HTTP_CLIENT_VERSION} (+https://rawdash.dev)`;
+}
+function parseEpoch(value, unit) {
+  if (value === null || value === void 0) {
+    return null;
+  }
+  if (unit === "iso") {
+    if (typeof value !== "string") {
+      return null;
+    }
+    const ms = new Date(value).getTime();
+    return Number.isFinite(ms) ? ms : null;
+  }
+  if (typeof value === "string" && value.trim() === "") {
+    return null;
+  }
+  const n = typeof value === "number" ? value : Number(value);
+  if (!Number.isFinite(n)) {
+    return null;
+  }
+  const result = unit === "s" ? n * 1e3 : n;
+  return Number.isFinite(result) ? result : null;
+}
+
+// src/langsmith.ts
+import {
+  BaseConnector,
+  defineConfigFields,
+  defineConnectorDoc,
+  defineResources,
+  makeChunkedCursorGuard,
+  paginateChunked,
+  schemasFromResources,
+  selectActivePhases
+} from "@rawdash/core";
+import { z } from "zod";
+var configFields = defineConfigFields(
+  z.object({
+    apiKey: z.object({ $secret: z.string() }).meta({
+      label: "API key",
+      description: "LangSmith API key with read access to the tenant. Create one in LangSmith -> Settings -> API Keys.",
+      placeholder: "lsv2_pt_...",
+      secret: true
+    }),
+    endpoint: z.string().trim().regex(
+      /^https?:\/\/[^\s/]+$/,
+      "Use a base URL with protocol and no trailing slash, e.g. https://api.smith.langchain.com"
+    ).default("https://api.smith.langchain.com").meta({
+      label: "Endpoint",
+      description: "LangSmith API base URL. Defaults to https://api.smith.langchain.com (US cloud). Use https://eu.api.smith.langchain.com for the EU region or your self-hosted origin. No trailing slash.",
+      placeholder: "https://api.smith.langchain.com"
+    }),
+    lookbackDays: z.number().int().positive().max(365).optional().meta({
+      label: "Lookback days (full sync)",
+      description: "How many calendar days of history to backfill on a full sync. Defaults to 30.",
+      placeholder: "30"
+    }),
+    resources: z.array(z.enum(["runs", "runs_per_day", "feedback"])).nonempty().optional().meta({
+      label: "Resources",
+      description: "Which LangSmith resources to sync. Omit to sync all of them. Both `runs` and `runs_per_day` are produced from the same upstream query, so listing either pulls runs."
+    })
+  })
+);
+var doc = defineConnectorDoc({
+  displayName: "LangSmith",
+  category: "engineering",
+  brandColor: "#7FC8FF",
+  tagline: "Sync LangChain runs, daily run rollups (count, tokens, cost, latency), and feedback scores from a LangSmith tenant.",
+  vendor: {
+    name: "LangSmith",
+    domain: "langchain.com",
+    apiDocs: "https://docs.smith.langchain.com/reference",
+    website: "https://smith.langchain.com"
+  },
+  auth: {
+    summary: "A LangSmith API key with read access is required. The key is sent as the `x-api-key` header on every request.",
+    setup: [
+      "Open LangSmith -> Settings -> API Keys and create a Personal Access Token (or Service key) with read access.",
+      "Copy the key (it is shown once).",
+      "Set `endpoint` to your LangSmith region: https://api.smith.langchain.com (US, default), https://eu.api.smith.langchain.com (EU), or your self-hosted origin (no trailing slash).",
+      'Store the API key as a secret and reference it from config as `apiKey: secret("LANGSMITH_API_KEY")`.'
+    ]
+  },
+  rateLimit: "LangSmith applies per-tenant rate limits and returns 429 with Retry-After on overrun; the shared HTTP client honors that header.",
+  limitations: [
+    "Run input/output payloads are not synced - only the run envelope plus aggregated cost, token, and latency.",
+    "Datasets, examples, prompts, and evaluation runs are out of scope for the initial release.",
+    "Feedback non-numeric values (string, boolean, JSON) are still counted but do not contribute to the score sample."
+  ]
+});
+var langsmithCredentials = {
+  apiKey: {
+    description: "LangSmith API key",
+    auth: "required"
+  }
+};
+var PHASE_ORDER = ["runs", "feedback"];
+var isLangSmithSyncCursor = makeChunkedCursorGuard(PHASE_ORDER);
+var RUNS_PAGE_SIZE = 100;
+var FEEDBACK_PAGE_SIZE = 100;
+var CHUNK_BUDGET_MS = 25e3;
+var DEFAULT_LOOKBACK_DAYS = 30;
+var MS_PER_DAY = 24 * 60 * 60 * 1e3;
+var RUN_ENTITY = "langsmith_run";
+var RUNS_PER_DAY_METRIC = "langsmith_runs_per_day";
+var FEEDBACK_METRIC = "langsmith_feedback";
+var runSchema = z.object({
+  id: z.string().min(1),
+  name: z.string().nullish(),
+  run_type: z.string().nullish(),
+  status: z.string().nullish(),
+  session_id: z.string().nullish(),
+  session_name: z.string().nullish(),
+  parent_run_id: z.string().nullish(),
+  start_time: z.string().nullish(),
+  end_time: z.string().nullish(),
+  error: z.string().nullish(),
+  total_tokens: z.number().nullish(),
+  prompt_tokens: z.number().nullish(),
+  completion_tokens: z.number().nullish(),
+  total_cost: z.number().nullish(),
+  prompt_cost: z.number().nullish(),
+  completion_cost: z.number().nullish(),
+  latency: z.number().nullish()
+});
+var runsQueryResponseSchema = z.object({
+  runs: z.array(runSchema),
+  cursors: z.object({
+    next: z.string().nullish()
+  }).nullish()
+});
+var feedbackSchema = z.object({
+  id: z.string().min(1),
+  run_id: z.string().nullish(),
+  session_id: z.string().nullish(),
+  key: z.string().min(1),
+  score: z.number().nullish(),
+  comment: z.string().nullish(),
+  created_at: z.string().nullish(),
+  modified_at: z.string().nullish()
+});
+var feedbackListResponseSchema = z.array(feedbackSchema);
+var langsmithResources = defineResources({
+  langsmith_run: {
+    shape: "entity",
+    filterable: [
+      {
+        field: "sessionId",
+        ops: ["eq"]
+      },
+      {
+        field: "runType",
+        ops: ["eq"],
+        values: ["chain", "tool", "llm", "embedding", "parser", "retriever"]
+      },
+      {
+        field: "status",
+        ops: ["eq"],
+        values: ["success", "error", "pending"]
+      }
+    ],
+    description: "LangSmith run rows, keyed by id, with name, owning session/project, parent run, run type, status, start/end timestamps, total/prompt/completion tokens, total/prompt/completion cost in USD, and end-to-end latency in milliseconds.",
+    endpoint: "POST /api/v1/runs/query",
+    notes: "Runs upsert by id on every run. Trace input/output payloads are not stored.",
+    fields: [
+      { name: "name", description: "Run name set by the SDK." },
+      {
+        name: "runType",
+        description: "Run type (chain, tool, llm, embedding, parser, retriever)."
+      },
+      {
+        name: "status",
+        description: "Run status (success, error, pending)."
+      },
+      {
+        name: "sessionId",
+        description: "Owning session (project) id, if any."
+      },
+      {
+        name: "sessionName",
+        description: "Owning session (project) name, if any."
+      },
+      {
+        name: "parentRunId",
+        description: "Parent run id for nested runs."
+      },
+      {
+        name: "startTime",
+        description: "ISO timestamp of run start."
+      },
+      {
+        name: "endTime",
+        description: "ISO timestamp of run end, if completed."
+      },
+      {
+        name: "totalTokens",
+        description: "Aggregate token count across the run."
+      },
+      {
+        name: "promptTokens",
+        description: "Prompt token count for the run."
+      },
+      {
+        name: "completionTokens",
+        description: "Completion token count for the run."
+      },
+      {
+        name: "totalCost",
+        description: "Aggregate run cost in USD."
+      },
+      {
+        name: "latencyMs",
+        description: "End-to-end latency in milliseconds."
+      },
+      {
+        name: "error",
+        description: "Error message if the run failed."
+      }
+    ],
+    responses: { runs: runsQueryResponseSchema }
+  },
+  langsmith_runs_per_day: {
+    shape: "metric",
+    description: "Per-run samples used to roll runs up to daily totals at query time. One sample is emitted per run at its start timestamp, tagged with project, run type, and status. The sample value is 1 (so summing yields the run count); token, cost, and latency are exposed as additional attribute fields.",
+    endpoint: "POST /api/v1/runs/query",
+    unit: "runs",
+    granularity: "Per-run (query-time rollup)",
+    notes: "No server-side aggregation - widgets group by day, project, or run type to produce the rollup.",
+    dimensions: [
+      {
+        name: "sessionId",
+        description: "Owning session (project) id, if any."
+      },
+      {
+        name: "sessionName",
+        description: "Owning session (project) name, if any."
+      },
+      {
+        name: "runType",
+        description: "Run type (chain, tool, llm, embedding, ...)."
+      },
+      {
+        name: "status",
+        description: "Run status (success, error, pending)."
+      },
+      {
+        name: "count",
+        description: "One per run; sum to get run counts. Also exposed as the sample value."
+      },
+      {
+        name: "totalTokens",
+        description: "Total tokens consumed by the run."
+      },
+      {
+        name: "promptTokens",
+        description: "Prompt tokens consumed by the run."
+      },
+      {
+        name: "completionTokens",
+        description: "Completion tokens produced by the run."
+      },
+      {
+        name: "costUsd",
+        description: "Aggregate run cost in USD."
+      },
+      {
+        name: "latencyMs",
+        description: "End-to-end run latency in milliseconds."
+      }
+    ],
+    responses: {}
+  },
+  langsmith_feedback: {
+    shape: "metric",
+    description: "Feedback rows from LangSmith, one sample per feedback row at its created_at timestamp. The sample value is the numeric score (zero for non-numeric feedback) and the attribute `count` is 1 so summing yields feedback counts per (day, project, key).",
+    endpoint: "GET /api/v1/feedback",
+    unit: "score",
+    granularity: "Per-feedback (query-time rollup)",
+    notes: "Non-numeric feedback (string, boolean, JSON value) is still emitted but with score 0; use `count` to count rows and average `score` for numeric trends.",
+    dimensions: [
+      {
+        name: "key",
+        description: "Feedback key as set by the SDK or annotator."
+      },
+      {
+        name: "sessionId",
+        description: "Owning session (project) id, if known."
+      },
+      {
+        name: "runId",
+        description: "Run the feedback is attached to, if any."
+      },
+      {
+        name: "count",
+        description: "One per feedback row; sum to count rows."
+      },
+      {
+        name: "score",
+        description: "Numeric score, or 0 for non-numeric feedback. Also exposed as the sample value."
+      },
+      {
+        name: "hasNumericScore",
+        description: "1 if the feedback row had a numeric score, 0 otherwise; sum these to compute strict numeric counts."
+      }
+    ],
+    responses: { feedback: feedbackListResponseSchema }
+  }
+});
+var id = "langsmith";
+var LangSmithConnector = class _LangSmithConnector extends BaseConnector {
+  static id = id;
+  static resources = langsmithResources;
+  static schemas = schemasFromResources(langsmithResources);
+  static create(input, ctx) {
+    const parsed = configFields.parse(input);
+    return new _LangSmithConnector(
+      {
+        endpoint: parsed.endpoint,
+        lookbackDays: parsed.lookbackDays,
+        resources: parsed.resources
+      },
+      { apiKey: parsed.apiKey },
+      ctx
+    );
+  }
+  id = id;
+  credentials = langsmithCredentials;
+  get baseUrl() {
+    return this.settings.endpoint.replace(/\/+$/, "");
+  }
+  buildHeaders(extra) {
+    return {
+      "x-api-key": this.creds.apiKey,
+      Accept: "application/json",
+      "User-Agent": connectorUserAgent("langsmith"),
+      ...extra ?? {}
+    };
+  }
+  windowStartIso(options) {
+    const lookbackDays = this.settings.lookbackDays ?? DEFAULT_LOOKBACK_DAYS;
+    const now = Date.now();
+    let startMs = now - lookbackDays * MS_PER_DAY;
+    if (options.since) {
+      const sinceMs = new Date(options.since).getTime();
+      if (Number.isFinite(sinceMs) && sinceMs < startMs) {
+        startMs = sinceMs;
+      }
+    }
+    return new Date(startMs).toISOString();
+  }
+  wantsRunEntity() {
+    return resourceIsActive(this.settings.resources, "runs");
+  }
+  wantsRunsPerDay() {
+    return resourceIsActive(this.settings.resources, "runs_per_day");
+  }
+  wantsFeedback() {
+    return resourceIsActive(this.settings.resources, "feedback");
+  }
+  async fetchRunsPage(options, page, signal) {
+    const offset = parseOffset(page);
+    const body = {
+      start_time: this.windowStartIso(options),
+      limit: RUNS_PAGE_SIZE,
+      offset,
+      order: "asc"
+    };
+    const res = await this.post(
+      `${this.baseUrl}/api/v1/runs/query`,
+      {
+        resource: "runs",
+        headers: this.buildHeaders({ "Content-Type": "application/json" }),
+        body: JSON.stringify(body),
+        signal
+      }
+    );
+    const runs = res.body.runs ?? [];
+    const sinceMs = options.since ? new Date(options.since).getTime() : null;
+    const allBeforeSince = sinceMs !== null && runs.length > 0 && runs.every((r) => {
+      const ts = parseEpoch(r.start_time ?? null, "iso");
+      return ts !== null && ts < sinceMs;
+    });
+    const explicitNext = res.body.cursors?.next ?? null;
+    let next;
+    if (allBeforeSince) {
+      next = null;
+    } else if (explicitNext) {
+      next = explicitNext;
+    } else if (runs.length === RUNS_PAGE_SIZE) {
+      next = String(offset + runs.length);
+    } else {
+      next = null;
+    }
+    return { items: runs, next };
+  }
+  async writeRunsBatch(storage, runs) {
+    const wantEntity = this.wantsRunEntity();
+    const wantMetric = this.wantsRunsPerDay();
+    if (!wantEntity && !wantMetric) {
+      return;
+    }
+    for (const run of runs) {
+      const startMs = parseEpoch(run.start_time ?? null, "iso");
+      const endMs = parseEpoch(run.end_time ?? null, "iso");
+      const latencyMs = computeLatencyMs(run, startMs, endMs);
+      if (wantEntity) {
+        const updatedAt = endMs ?? startMs ?? 0;
+        await storage.entity({
+          type: RUN_ENTITY,
+          id: run.id,
+          attributes: {
+            name: run.name ?? null,
+            runType: run.run_type ?? null,
+            status: run.status ?? null,
+            sessionId: run.session_id ?? null,
+            sessionName: run.session_name ?? null,
+            parentRunId: run.parent_run_id ?? null,
+            startTime: run.start_time ?? null,
+            endTime: run.end_time ?? null,
+            totalTokens: finiteNumberOrNull(run.total_tokens),
+            promptTokens: finiteNumberOrNull(run.prompt_tokens),
+            completionTokens: finiteNumberOrNull(run.completion_tokens),
+            totalCost: finiteNumberOrNull(run.total_cost),
+            promptCost: finiteNumberOrNull(run.prompt_cost),
+            completionCost: finiteNumberOrNull(run.completion_cost),
+            latencyMs,
+            error: run.error ?? null
+          },
+          updated_at: updatedAt
+        });
+      }
+      if (wantMetric && startMs !== null) {
+        await storage.metric({
+          name: RUNS_PER_DAY_METRIC,
+          ts: startMs,
+          value: 1,
+          attributes: {
+            sessionId: run.session_id ?? null,
+            sessionName: run.session_name ?? null,
+            runType: run.run_type ?? null,
+            status: run.status ?? null,
+            count: 1,
+            totalTokens: finiteNumber(run.total_tokens),
+            promptTokens: finiteNumber(run.prompt_tokens),
+            completionTokens: finiteNumber(run.completion_tokens),
+            costUsd: finiteNumber(run.total_cost),
+            latencyMs: latencyMs ?? 0
+          }
+        });
+      }
+    }
+  }
+  async fetchFeedbackPage(options, page, signal) {
+    const offset = parseOffset(page);
+    const url = new URL(`${this.baseUrl}/api/v1/feedback`);
+    url.searchParams.set("limit", String(FEEDBACK_PAGE_SIZE));
+    url.searchParams.set("offset", String(offset));
+    url.searchParams.set("start_time", this.windowStartIso(options));
+    const res = await this.get(url.toString(), {
+      resource: "feedback",
+      headers: this.buildHeaders(),
+      signal
+    });
+    const rows = Array.isArray(res.body) ? res.body : [];
+    const sinceMs = options.since ? new Date(options.since).getTime() : null;
+    const allBeforeSince = sinceMs !== null && rows.length > 0 && rows.every((f) => {
+      const ts = parseEpoch(f.created_at ?? null, "iso");
+      return ts !== null && ts < sinceMs;
+    });
+    const next = !allBeforeSince && rows.length === FEEDBACK_PAGE_SIZE ? String(offset + rows.length) : null;
+    return { items: rows, next };
+  }
+  async writeFeedbackBatch(storage, rows) {
+    for (const row of rows) {
+      const ts = parseEpoch(row.created_at ?? null, "iso");
+      if (ts === null) {
+        continue;
+      }
+      const numeric = typeof row.score === "number" && Number.isFinite(row.score);
+      const score = numeric ? row.score : 0;
+      await storage.metric({
+        name: FEEDBACK_METRIC,
+        ts,
+        value: score,
+        attributes: {
+          key: row.key,
+          sessionId: row.session_id ?? null,
+          runId: row.run_id ?? null,
+          count: 1,
+          score,
+          hasNumericScore: numeric ? 1 : 0
+        }
+      });
+    }
+  }
+  activePhases() {
+    return selectActivePhases(
+      (r) => {
+        switch (r) {
+          case "runs":
+          case "runs_per_day":
+            return "runs";
+          case "feedback":
+            return "feedback";
+        }
+      },
+      PHASE_ORDER,
+      this.settings.resources
+    );
+  }
+  async clearScopeOnFirstPage(storage, phase, isFull) {
+    switch (phase) {
+      case "runs":
+        if (isFull && this.wantsRunEntity()) {
+          await storage.entities([], { types: [RUN_ENTITY] });
+        }
+        if (this.wantsRunsPerDay()) {
+          await storage.metrics([], { names: [RUNS_PER_DAY_METRIC] });
+        }
+        return;
+      case "feedback":
+        await storage.metrics([], { names: [FEEDBACK_METRIC] });
+        return;
+    }
+  }
+  async sync(options, storage, signal) {
+    const cursor = isLangSmithSyncCursor(options.cursor) ? options.cursor : void 0;
+    const isFull = options.mode === "full";
+    const phases = this.activePhases();
+    return paginateChunked({
+      phases,
+      cursor,
+      signal,
+      logger: this.logger,
+      maxChunkMs: CHUNK_BUDGET_MS,
+      fetchPage: async (phase, page, sig) => {
+        switch (phase) {
+          case "runs":
+            return this.fetchRunsPage(options, page, sig);
+          case "feedback":
+            return this.fetchFeedbackPage(options, page, sig);
+        }
+      },
+      writeBatch: async (phase, items, page) => {
+        if (page === null) {
+          await this.clearScopeOnFirstPage(storage, phase, isFull);
+        }
+        switch (phase) {
+          case "runs":
+            await this.writeRunsBatch(storage, items);
+            return;
+          case "feedback":
+            if (this.wantsFeedback()) {
+              await this.writeFeedbackBatch(storage, items);
+            }
+            return;
+        }
+      }
+    });
+  }
+};
+function parseOffset(page) {
+  if (page === null) {
+    return 0;
+  }
+  const n = Number(page);
+  return Number.isFinite(n) && n >= 0 ? Math.floor(n) : 0;
+}
+function finiteNumber(value, fallback = 0) {
+  if (value === null || value === void 0) {
+    return fallback;
+  }
+  const n = typeof value === "number" ? value : Number(value);
+  return Number.isFinite(n) ? n : fallback;
+}
+function finiteNumberOrNull(value) {
+  if (value === null || value === void 0 || value === "") {
+    return null;
+  }
+  const n = typeof value === "number" ? value : Number(value);
+  return Number.isFinite(n) ? n : null;
+}
+function computeLatencyMs(run, startMs, endMs) {
+  if (typeof run.latency === "number" && Number.isFinite(run.latency)) {
+    return run.latency;
+  }
+  if (startMs !== null && endMs !== null && endMs >= startMs) {
+    return endMs - startMs;
+  }
+  return null;
+}
+function resourceIsActive(allowlist, resource) {
+  if (!allowlist || allowlist.length === 0) {
+    return true;
+  }
+  return allowlist.includes(resource);
+}
+
+// src/index.ts
+var index_default = LangSmithConnector;
+export {
+  LangSmithConnector,
+  configFields,
+  index_default as default,
+  doc,
+  id,
+  langsmithResources as resources
+};
+//# sourceMappingURL=index.js.map