@ciphyrshq/sdk 2.6.0

package/src/errors.js

@@ -0,0 +1,52 @@
+export class CiphyrsError extends Error {
+  constructor(message, { status, code } = {}) {
+    super(message);
+    this.name   = 'CiphyrsError';
+    this.status = status;
+    this.code   = code;
+  }
+}
+
+export class CiphyrsAuthError extends CiphyrsError {
+  constructor(message = 'Unauthorized') {
+    super(message, { status: 401, code: 'AUTH_ERROR' });
+    this.name = 'CiphyrsAuthError';
+  }
+}
+
+export class CiphyrsPermissionError extends CiphyrsError {
+  constructor(message = 'Forbidden') {
+    super(message, { status: 403, code: 'PERMISSION_ERROR' });
+    this.name = 'CiphyrsPermissionError';
+  }
+}
+
+export class CiphyrsNotFoundError extends CiphyrsError {
+  constructor(message = 'Not found') {
+    super(message, { status: 404, code: 'NOT_FOUND' });
+    this.name = 'CiphyrsNotFoundError';
+  }
+}
+
+export class CiphyrsRateLimitError extends CiphyrsError {
+  constructor(message = 'Rate limit exceeded', { retryAfter } = {}) {
+    super(message, { status: 429, code: 'RATE_LIMIT' });
+    this.name        = 'CiphyrsRateLimitError';
+    this.retryAfter  = retryAfter ? parseInt(retryAfter) : null;
+  }
+}
+
+export class CiphyrsTimeoutError extends CiphyrsError {
+  constructor(message = 'Request timed out') {
+    super(message, { code: 'TIMEOUT' });
+    this.name = 'CiphyrsTimeoutError';
+  }
+}
+
+export class CiphyrsJobTimeoutError extends CiphyrsError {
+  constructor(jobId, timeoutMs) {
+    super(`Job ${jobId} did not complete within ${timeoutMs}ms`, { code: 'JOB_TIMEOUT' });
+    this.name  = 'CiphyrsJobTimeoutError';
+    this.jobId = jobId;
+  }
+}

package/src/eval-runner.js

@@ -0,0 +1,127 @@
+// ─────────────────────────────────────────────────────────────────────────────
+// EvalRunner — run evaluation datasets against agent functions
+// Port of the Python SDK's EvalRunner
+// ─────────────────────────────────────────────────────────────────────────────
+
+export class EvalRunner {
+  #client;
+
+  /**
+   * @param {import('./client.js').CiphyrsClient} client
+   */
+  constructor(client) {
+    this.#client = client;
+  }
+
+  /**
+   * Load a dataset by ID from the Ciphyrs eval service.
+   * @param {string} datasetId
+   * @returns {Promise<object>}
+   */
+  async loadDataset(datasetId) {
+    return this.#client._request(
+      `${this.#client._baseUrl}/v1/evals/datasets/${encodeURIComponent(datasetId)}`,
+    );
+  }
+
+  /**
+   * Run an agent function against every test case in a dataset.
+   *
+   * @param {(input: any) => Promise<any>} agentFn — The agent under test
+   * @param {string} datasetId — Dataset to evaluate against
+   * @param {object} [opts]
+   * @param {(input: any, expected: any, actual: any) => Promise<number>} [opts.judgeFn]
+   *   Custom judge that returns a score 0-1. If omitted, strict equality is used.
+   * @param {string} [opts.agentName]    — Name of the agent being evaluated
+   * @param {string} [opts.agentVersion] — Version string for the agent
+   * @returns {Promise<{run_id: string, score: number, passed: number, failed: number, total: number, duration_ms: number, results: Array}>}
+   */
+  async run(agentFn, datasetId, { judgeFn, agentName, agentVersion } = {}) {
+    const { dataset } = await this.loadDataset(datasetId);
+    const cases = dataset.test_cases || [];
+
+    // Create eval run on the backend
+    const { run: evalRun } = await this.#client._request(
+      `${this.#client._baseUrl}/v1/evals/runs`,
+      {
+        method: 'POST',
+        body: {
+          dataset_id: datasetId,
+          agent_name: agentName,
+          agent_version: agentVersion,
+          total_cases: cases.length,
+        },
+      },
+    );
+
+    const results = [];
+    let passed = 0;
+    let failed = 0;
+    let totalScore = 0;
+    const startTime = Date.now();
+
+    for (const testCase of cases) {
+      const caseStart = Date.now();
+      try {
+        const actual = await agentFn(testCase.input);
+        const score = judgeFn
+          ? await judgeFn(testCase.input, testCase.expected, actual)
+          : (actual === testCase.expected ? 1.0 : 0.0);
+        const pass = score >= (testCase.threshold || 0.5);
+
+        results.push({
+          input: testCase.input,
+          expected: testCase.expected,
+          actual,
+          score,
+          passed: pass,
+          latency_ms: Date.now() - caseStart,
+        });
+
+        if (pass) passed++;
+        else failed++;
+        totalScore += score;
+      } catch (err) {
+        results.push({
+          input: testCase.input,
+          expected: testCase.expected,
+          actual: null,
+          score: 0,
+          passed: false,
+          error: err.message,
+          latency_ms: Date.now() - caseStart,
+        });
+        failed++;
+      }
+    }
+
+    const avgScore = cases.length > 0 ? totalScore / cases.length : 0;
+
+    // Update the eval run with results
+    await this.#client._request(
+      `${this.#client._baseUrl}/v1/evals/runs/${encodeURIComponent(evalRun.id)}`,
+      {
+        method: 'PATCH',
+        body: {
+          status: 'completed',
+          passed,
+          failed,
+          avg_score: avgScore,
+          avg_latency_ms: cases.length > 0 ? Math.round((Date.now() - startTime) / cases.length) : 0,
+          results,
+          completed_at: new Date().toISOString(),
+        },
+      },
+    );
+
+    return {
+      run_id: evalRun.id,
+      score: avgScore,
+      passed,
+      failed,
+      total: cases.length,
+      duration_ms: Date.now() - startTime,
+      results,
+    };
+  }
+}

package/src/index.js

@@ -0,0 +1,13 @@
+export { CiphyrsClient } from './client.js';
+export {
+  CiphyrsError,
+  CiphyrsAuthError,
+  CiphyrsPermissionError,
+  CiphyrsNotFoundError,
+  CiphyrsRateLimitError,
+  CiphyrsTimeoutError,
+  CiphyrsJobTimeoutError,
+} from './errors.js';
+export { CiphyrsTracer, Trace, Span } from './tracer.js';
+export { SecretDetector } from './secret-detector.js';
+export { EvalRunner } from './eval-runner.js';

package/src/secret-detector.js

@@ -0,0 +1,49 @@
+// ─────────────────────────────────────────────────────────────────────────────
+// SecretDetector — scan text for leaked credentials and API keys
+// Port of the Python SDK's SecretDetector
+// ─────────────────────────────────────────────────────────────────────────────
+
+const PATTERNS = [
+  { type: 'aws_access_key',  pattern: /AKIA[0-9A-Z]{16}/g,                                                    confidence: 0.95 },
+  { type: 'aws_secret_key',  pattern: /[A-Za-z0-9/+=]{40}(?=\s|$|")/g,                                        confidence: 0.7  },
+  { type: 'github_token',    pattern: /gh[psortu]_[A-Za-z0-9_]{36,}/g,                                        confidence: 0.99 },
+  { type: 'gitlab_token',    pattern: /glpat-[A-Za-z0-9\-_]{20,}/g,                                           confidence: 0.95 },
+  { type: 'slack_token',     pattern: /xox[bpras]-[A-Za-z0-9\-]+/g,                                           confidence: 0.95 },
+  { type: 'stripe_key',      pattern: /[sr]k_(test|live)_[A-Za-z0-9]{24,}/g,                                  confidence: 0.99 },
+  { type: 'generic_api_key', pattern: /(?:api[_-]?key|apikey|token|secret)['":\s]*[=:]\s*['"]?([A-Za-z0-9_\-]{20,})['"]?/gi, confidence: 0.6 },
+  { type: 'jwt',             pattern: /eyJ[A-Za-z0-9_-]+\.eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+/g,               confidence: 0.9  },
+  { type: 'private_key',     pattern: /-----BEGIN (?:RSA |EC |DSA )?PRIVATE KEY-----/g,                        confidence: 0.99 },
+];
+
+export class SecretDetector {
+  /**
+   * Scan text for potential secrets and credentials.
+   * @param {string} text — The text to scan
+   * @returns {Array<{type: string, start: number, end: number, value_masked: string, confidence: number}>}
+   */
+  detect(text) {
+    if (!text || typeof text !== 'string') return [];
+
+    const findings = [];
+
+    for (const { type, pattern, confidence } of PATTERNS) {
+      // Clone the regex so lastIndex resets for each call
+      const re = new RegExp(pattern.source, pattern.flags);
+      let match;
+      while ((match = re.exec(text)) !== null) {
+        const value = match[0];
+        findings.push({
+          type,
+          start: match.index,
+          end: match.index + value.length,
+          value_masked: value.length > 8
+            ? value.slice(0, 4) + '***' + value.slice(-4)
+            : value.slice(0, 2) + '***',
+          confidence,
+        });
+      }
+    }
+
+    return findings;
+  }
+}

package/src/tracer.js

@@ -0,0 +1,220 @@
+import { randomUUID } from 'node:crypto';
+
+// ─────────────────────────────────────────────────────────────────────────────
+// CiphyrsTracer — buffered span/metric collection with auto-flush
+// Mirrors the Python SDK's tracer.py
+// ─────────────────────────────────────────────────────────────────────────────
+
+export class CiphyrsTracer {
+  #client;
+  #projectName;
+  #config;
+  #spanBuffer = [];
+  #metricBuffer = [];
+  #flushInterval;
+
+  /**
+   * @param {import('./client.js').CiphyrsClient} client
+   * @param {object} opts
+   * @param {string}  opts.projectName      — Project name for grouping traces
+   * @param {number}  [opts.flushIntervalMs=5000] — Auto-flush interval in ms
+   * @param {number}  [opts.batchSize=50]   — Flush when buffer reaches this size
+   */
+  constructor(client, { projectName, flushIntervalMs = 5000, batchSize = 50 } = {}) {
+    this.#client = client;
+    this.#projectName = projectName;
+    this.#config = { flushIntervalMs, batchSize };
+    this.#flushInterval = setInterval(() => this.flush().catch(() => {}), flushIntervalMs);
+    // Prevent the interval from keeping the process alive
+    if (this.#flushInterval.unref) this.#flushInterval.unref();
+  }
+
+  /**
+   * Start a new trace.
+   * @param {string} name — Human-readable trace name
+   * @param {object} [opts]
+   * @param {string} [opts.traceId] — Provide your own trace ID, or one is generated
+   * @returns {Trace}
+   */
+  trace(name, opts = {}) {
+    return new Trace(this, name, opts);
+  }
+
+  /** @internal — called by Span.end() to enqueue completed spans */
+  _enqueueSpan(span) {
+    this.#spanBuffer.push(span);
+    if (this.#spanBuffer.length >= this.#config.batchSize) {
+      this.flush().catch(() => {});
+    }
+  }
+
+  /**
+   * Record a custom metric.
+   * @param {string} name  — Metric name (e.g. "agent.latency")
+   * @param {number} value — Metric value
+   * @param {object} [opts]
+   * @param {string} [opts.unit]      — Unit label (e.g. "ms", "usd")
+   * @param {Record<string,string>} [opts.tags] — Key-value tags
+   * @param {string} [opts.agentName] — Agent that emitted the metric
+   */
+  emitMetric(name, value, { unit, tags, agentName } = {}) {
+    this.#metricBuffer.push({ name, value, unit, tags, agent_name: agentName });
+  }
+
+  /**
+   * Flush buffered spans and metrics to the Ciphyrs backend.
+   * Called automatically on the flush interval, or when batchSize is reached.
+   */
+  async flush() {
+    if (!this.#spanBuffer.length && !this.#metricBuffer.length) return;
+
+    const spans = this.#spanBuffer.splice(0);
+    const metrics = this.#metricBuffer.splice(0);
+
+    const promises = [];
+
+    if (spans.length) {
+      // Group spans by trace_id so each ingest call contains one trace
+      const byTrace = {};
+      for (const s of spans) {
+        (byTrace[s.trace_id] ||= []).push(s);
+      }
+      for (const [traceId, traceSpans] of Object.entries(byTrace)) {
+        promises.push(
+          this.#client.trace.ingest(
+            { name: this.#projectName },
+            { trace_id: traceId, name: traceSpans[0]?.trace_name || traceId },
+            traceSpans,
+          ).catch(err => console.error('Ciphyrs flush error:', err.message)),
+        );
+      }
+    }
+
+    if (metrics.length) {
+      promises.push(
+        this.#client._request(`${this.#client._baseUrl}/v1/observe/metrics`, {
+          method: 'POST',
+          body: { metrics },
+        }).catch(err => console.error('Ciphyrs metrics flush error:', err.message)),
+      );
+    }
+
+    await Promise.allSettled(promises);
+  }
+
+  /**
+   * Stop the auto-flush timer and flush any remaining data.
+   * Always call this before process exit to avoid data loss.
+   */
+  async shutdown() {
+    clearInterval(this.#flushInterval);
+    await this.flush();
+  }
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Trace — groups related spans under a single trace ID
+// ─────────────────────────────────────────────────────────────────────────────
+
+export class Trace {
+  #tracer;
+  #traceId;
+  #name;
+  #startTime;
+
+  constructor(tracer, name, opts = {}) {
+    this.#tracer = tracer;
+    this.#traceId = opts.traceId || randomUUID();
+    this.#name = name;
+    this.#startTime = Date.now();
+  }
+
+  get traceId() { return this.#traceId; }
+  get name()    { return this.#name; }
+
+  /**
+   * Create a new span within this trace.
+   * @param {string} name — Operation name for the span
+   * @param {object} [opts]
+   * @param {string} [opts.parentSpanId] — Parent span ID for nested spans
+   * @param {string} [opts.agentName]    — Agent performing this operation
+   * @param {string} [opts.kind]         — Span kind: 'agent' | 'tool' | 'llm' | 'retriever'
+   * @returns {Span}
+   */
+  span(name, opts = {}) {
+    return new Span(this.#tracer, this.#traceId, this.#name, name, opts);
+  }
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Span — a single timed operation within a trace
+// ─────────────────────────────────────────────────────────────────────────────
+
+export class Span {
+  #tracer;
+  #data;
+  #startTime;
+
+  constructor(tracer, traceId, traceName, name, opts = {}) {
+    this.#tracer = tracer;
+    this.#startTime = Date.now();
+    this.#data = {
+      span_id: randomUUID(),
+      trace_id: traceId,
+      trace_name: traceName,
+      parent_span_id: opts.parentSpanId || null,
+      agent_name: opts.agentName || null,
+      kind: opts.kind || 'agent',
+      operation_name: name,
+      input_text: null,
+      output_text: null,
+      status: 'ok',
+      cost_usd: null,
+      model_name: null,
+      prompt_tokens: null,
+      completion_tokens: null,
+    };
+  }
+
+  get spanId() { return this.#data.span_id; }
+
+  /** Set the input text for this span. */
+  setInput(text)  { this.#data.input_text = text; return this; }
+
+  /** Set the output text for this span. */
+  setOutput(text) { this.#data.output_text = text; return this; }
+
+  /** Set span status ('ok' | 'error'). */
+  setStatus(s)    { this.#data.status = s; return this; }
+
+  /** Mark span as errored with a message. */
+  setError(msg)   { this.#data.status = 'error'; this.#data.error_message = msg; return this; }
+
+  /** Set the LLM cost in USD. */
+  setCost(usd)    { this.#data.cost_usd = usd; return this; }
+
+  /** Set the model name (e.g. 'gpt-4', 'claude-3-opus'). */
+  setModel(name)  { this.#data.model_name = name; return this; }
+
+  /** Set prompt and completion token counts. */
+  setTokens(prompt, completion) {
+    this.#data.prompt_tokens = prompt;
+    this.#data.completion_tokens = completion;
+    return this;
+  }
+
+  /** Attach arbitrary metadata / attributes. */
+  setMetadata(meta) { this.#data.attributes = meta; return this; }
+
+  /**
+   * End the span, recording timestamps and duration, then enqueue for flush.
+   * @returns {this}
+   */
+  end() {
+    this.#data.started_at = new Date(this.#startTime).toISOString();
+    this.#data.ended_at = new Date().toISOString();
+    this.#data.duration_ms = Date.now() - this.#startTime;
+    this.#tracer._enqueueSpan(this.#data);
+    return this;
+  }
+}