braintrust 0.0.89 → 0.0.91

package/dist/index.js

@@ -7766,7 +7766,9 @@ async function getRepoStatus() {
     async () => (await git.raw(["rev-parse", "--abbrev-ref", "HEAD"])).trim()
   );
   if (dirty) {
-    git_diff = await attempt(async () => truncateToByteLimit(await git.raw(["diff", "HEAD"])));
+    git_diff = await attempt(
+      async () => truncateToByteLimit(await git.raw(["diff", "HEAD"]))
+    );
   }
   return {
     commit,
@@ -7875,6 +7877,10 @@ var v4_default = v4;
 // ../core/js/dist/index.mjs
 var TRANSACTION_ID_FIELD = "_xact_id";
 var IS_MERGE_FIELD = "_is_merge";
+var AUDIT_SOURCE_FIELD = "_audit_source";
+var AUDIT_METADATA_FIELD = "_audit_metadata";
+var VALID_SOURCES = ["app", "api", "external"];
+var PARENT_ID_FIELD = "_parent_id";
 function mergeDicts(mergeInto, mergeFrom) {
   for (const [k, mergeFromV] of Object.entries(mergeFrom)) {
     const mergeIntoV = mergeInto[k];
@@ -7887,6 +7893,7 @@ function mergeDicts(mergeInto, mergeFrom) {
       mergeInto[k] = mergeFromV;
     }
   }
+  return mergeInto;
 }
 function generateMergedRowKey(row) {
   return JSON.stringify(
@@ -7928,6 +7935,15 @@ function mergeRowBatch(rows) {
   out.push(...Object.values(rowGroups));
   return out;
 }
+var SpanTypeAttribute = /* @__PURE__ */ ((SpanTypeAttribute2) => {
+  SpanTypeAttribute2["LLM"] = "llm";
+  SpanTypeAttribute2["SCORE"] = "score";
+  SpanTypeAttribute2["FUNCTION"] = "function";
+  SpanTypeAttribute2["EVAL"] = "eval";
+  SpanTypeAttribute2["TASK"] = "task";
+  SpanTypeAttribute2["TOOL"] = "tool";
+  return SpanTypeAttribute2;
+})(SpanTypeAttribute || {});
 
 // src/util.ts
 var GLOBAL_PROJECT = "Global";
@@ -7950,6 +7966,9 @@ function runFinally(f, finallyF) {
 function getCurrentUnixTimestamp() {
   return (/* @__PURE__ */ new Date()).getTime() / 1e3;
 }
+function isEmpty(a) {
+  return a === void 0 || a === null;
+}
 
 // src/logger.ts
 var NoopSpan = class {
@@ -7961,6 +7980,8 @@ var NoopSpan = class {
   }
   log(_) {
   }
+  logFeedback(event) {
+  }
   traced(callback, _1) {
     return callback(this);
   }
@@ -8135,6 +8156,70 @@ var HTTPConnection = class _HTTPConnection {
     return await resp.json();
   }
 };
+function logFeedbackImpl(bgLogger, parentIds, {
+  id,
+  expected,
+  scores,
+  metadata: inputMetadata,
+  comment,
+  source: inputSource
+}) {
+  const source = inputSource ?? "external";
+  if (!VALID_SOURCES.includes(source)) {
+    throw new Error(`source must be one of ${VALID_SOURCES}`);
+  }
+  if (isEmpty(scores) && isEmpty(expected) && isEmpty(comment)) {
+    throw new Error(
+      "At least one of scores, expected, or comment must be specified"
+    );
+  }
+  const validatedEvent = validateAndSanitizeExperimentLogPartialArgs({
+    scores,
+    metadata: inputMetadata,
+    expected
+  });
+  let { metadata, ...updateEvent } = validatedEvent;
+  updateEvent = Object.fromEntries(
+    Object.entries(updateEvent).filter(([_, v]) => !isEmpty(v))
+  );
+  const trueParentIds = (async () => {
+    const { kind, ...ids } = await parentIds;
+    return ids;
+  })();
+  if (Object.keys(updateEvent).length > 0) {
+    const record = (async () => {
+      return {
+        id,
+        ...updateEvent,
+        ...await trueParentIds,
+        [AUDIT_SOURCE_FIELD]: source,
+        [AUDIT_METADATA_FIELD]: metadata,
+        [IS_MERGE_FIELD]: true
+      };
+    })();
+    bgLogger.log([record]);
+  }
+  if (!isEmpty(comment)) {
+    const record = (async () => {
+      return {
+        id: v4_default(),
+        created: (/* @__PURE__ */ new Date()).toISOString(),
+        origin: {
+          // NOTE: We do not know (or care?) what the transaction id of the row that
+          // we're commenting on is here, so we omit it.
+          id
+        },
+        comment: {
+          text: comment
+        },
+        ...await trueParentIds,
+        [AUDIT_SOURCE_FIELD]: source,
+        [AUDIT_METADATA_FIELD]: metadata
+      };
+    })();
+    bgLogger.log([record]);
+  }
+}
 var Logger = class {
   constructor(lazyMetadata, logOptions = {}) {
     // For type identification.
@@ -8213,6 +8298,14 @@ var Logger = class {
       })();
     }
   }
+  async lazyParentIds() {
+    return {
+      kind: "project_log",
+      org_id: await this.org_id,
+      project_id: (await this.project).id,
+      log_id: "g"
+    };
+  }
   /**
    * Lower-level alternative to `traced`, which does not automatically end the span or mark it as current.
    *
@@ -8220,19 +8313,27 @@ var Logger = class {
    */
   startSpan(args) {
     const { name, ...argsRest } = args ?? {};
-    const parentIds = (async () => ({
-      kind: "project_log",
-      org_id: await this.org_id,
-      project_id: (await this.project).id,
-      log_id: "g"
-    }))();
     return new SpanImpl({
-      parentIds,
+      parentIds: this.lazyParentIds(),
       bgLogger: this.bgLogger,
       name: name ?? "root",
       ...argsRest
     });
   }
+  /**
+   * Log feedback to an event. Feedback is used to save feedback scores, set an expected value, or add a comment.
+   *
+   * @param event
+   * @param event.id The id of the event to log feedback for. This is the `id` returned by `log` or accessible as the `id` field of a span.
+   * @param event.scores (Optional) a dictionary of numeric values (between 0 and 1) to log. These scores will be merged into the existing scores for the event.
+   * @param event.expected (Optional) the ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to determine if your `output` value is correct or not.
+   * @param event.comment (Optional) an optional comment string to log about the event.
+   * @param event.metadata (Optional) a dictionary with additional data about the feedback. If you have a `user_id`, you can log it here and access it in the Braintrust UI.
+   * @param event.source (Optional) the source of the feedback. Must be one of "external" (default), "app", or "api".
+   */
+  logFeedback(event) {
+    logFeedbackImpl(this.bgLogger, this.lazyParentIds(), event);
+  }
   /*
    * Flush any pending logs to the server.
    */
@@ -8824,6 +8925,13 @@ var Experiment = class {
       () => span.end()
     );
   }
+  async lazyParentIds() {
+    return {
+      kind: "experiment",
+      project_id: (await this.project).id,
+      experiment_id: await this.id
+    };
+  }
   /**
    * Lower-level alternative to `traced`, which does not automatically end the span or mark it as current.
    *
@@ -8831,13 +8939,8 @@ var Experiment = class {
    */
   startSpan(args) {
     const { name, ...argsRest } = args ?? {};
-    const parentIds = (async () => ({
-      kind: "experiment",
-      project_id: (await this.project).id,
-      experiment_id: await this.id
-    }))();
     return new SpanImpl({
-      parentIds,
+      parentIds: this.lazyParentIds(),
       bgLogger: this.bgLogger,
       name: name ?? "root",
       ...argsRest
@@ -8899,6 +9002,20 @@ var Experiment = class {
       metrics
     };
   }
+  /**
+   * Log feedback to an event in the experiment. Feedback is used to save feedback scores, set an expected value, or add a comment.
+   *
+   * @param event
+   * @param event.id The id of the event to log feedback for. This is the `id` returned by `log` or accessible as the `id` field of a span.
+   * @param event.scores (Optional) a dictionary of numeric values (between 0 and 1) to log. These scores will be merged into the existing scores for the event.
+   * @param event.expected (Optional) the ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to determine if your `output` value is correct or not.
+   * @param event.comment (Optional) an optional comment string to log about the event.
+   * @param event.metadata (Optional) a dictionary with additional data about the feedback. If you have a `user_id`, you can log it here and access it in the Braintrust UI.
+   * @param event.source (Optional) the source of the feedback. Must be one of "external" (default), "app", or "api".
+   */
+  logFeedback(event) {
+    logFeedbackImpl(this.bgLogger, this.lazyParentIds(), event);
+  }
   /**
    * Flush any pending rows to the server.
    */
@@ -8949,10 +9066,12 @@ var SpanImpl = class _SpanImpl {
     this.rowIds = {
       id,
       span_id,
-      root_span_id: args.parentSpanInfo?.root_span_id ?? span_id
+      root_span_id: "parentSpanInfo" in args && args.parentSpanInfo?.root_span_id ? args.parentSpanInfo.root_span_id : span_id
     };
-    if (args.parentSpanInfo) {
+    if ("parentSpanInfo" in args && args.parentSpanInfo?.span_id) {
       this.internalData.span_parents = [args.parentSpanInfo.span_id];
+    } else if ("parentId" in args && !isEmpty(args.parentId)) {
+      this.rowIds[PARENT_ID_FIELD] = args.parentId;
     }
     this.isMerge = false;
     const { id: _id, ...eventRest } = args.event ?? {};
@@ -8976,16 +9095,26 @@ var SpanImpl = class _SpanImpl {
     if (sanitizedAndInternalData.metrics?.end) {
       this.loggedEndTime = sanitizedAndInternalData.metrics?.end;
     }
+    const parentIds = (async () => {
+      const { kind, ...ids } = await this.parentIds;
+      return ids;
+    })();
     const record = (async () => {
       return {
         ...sanitizedAndInternalData,
         ...this.rowIds,
-        ...await this.parentIds,
+        ...await parentIds,
         [IS_MERGE_FIELD]: this.isMerge
       };
     })();
     this.bgLogger.log([record]);
   }
+  logFeedback(event) {
+    logFeedbackImpl(this.bgLogger, this.parentIds, {
+      ...event,
+      id: this.id
+    });
+  }
   traced(callback, args) {
     const { setCurrent, ...argsRest } = args ?? {};
     const span = this.startSpan(argsRest);
@@ -9375,7 +9504,7 @@ async function runEvaluator(experiment, evaluator, progressReporter, filters) {
             }
             span.log({ input: datum.input, output });
           },
-          { name: "task" }
+          { name: "task", spanAttributes: { type: SpanTypeAttribute.TASK } }
         );
         rootSpan.log({ output });
         const scoringArgs = { ...datum, metadata, output };
@@ -9398,6 +9527,9 @@ async function runEvaluator(experiment, evaluator, progressReporter, filters) {
               },
               {
                 name: score.name || `scorer_${score_idx}`,
+                spanAttributes: {
+                  type: SpanTypeAttribute.SCORE
+                },
                 event: { input: scoringArgs }
               }
             );
@@ -9437,6 +9569,9 @@ async function runEvaluator(experiment, evaluator, progressReporter, filters) {
     } else {
       return await experiment.traced(callback, {
         name: "eval",
+        spanAttributes: {
+          type: SpanTypeAttribute.EVAL
+        },
         event: {
           input: datum.input,
           expected: datum.expected
@@ -9532,6 +9667,7 @@ function wrapOpenAI(openai) {
     return openai;
   }
 }
+globalThis.__inherited_braintrust_wrap_openai = wrapOpenAI;
 function wrapOpenAIv4(openai) {
   let completionProxy = new Proxy(openai.chat.completions, {
     get(target, name, receiver) {
@@ -9550,6 +9686,15 @@ function wrapOpenAIv4(openai) {
       return Reflect.get(target, name, receiver);
     }
   });
+  let embeddingProxy = new Proxy(openai.embeddings, {
+    get(target, name, receiver) {
+      const baseVal = Reflect.get(target, name, receiver);
+      if (name === "create") {
+        return wrapEmbeddings(baseVal.bind(target));
+      }
+      return baseVal;
+    }
+  });
   let betaProxy;
   if (openai.beta?.chat?.completions?.stream) {
     let betaChatCompletionProxy = new Proxy(openai?.beta?.chat.completions, {
@@ -9583,6 +9728,9 @@ function wrapOpenAIv4(openai) {
       if (name === "chat") {
         return chatProxy;
       }
+      if (name === "embeddings") {
+        return embeddingProxy;
+      }
       if (name === "beta" && betaProxy) {
         return betaProxy;
       }
@@ -9592,10 +9740,13 @@ function wrapOpenAIv4(openai) {
   return proxy;
 }
 function wrapBetaChatCompletion(completion) {
-  return async (params) => {
+  return (params) => {
     const { messages, ...rest } = params;
     const span = startSpan({
       name: "OpenAI Chat Completion",
+      spanAttributes: {
+        type: SpanTypeAttribute.LLM
+      },
       event: {
         input: messages,
         metadata: {
@@ -9604,7 +9755,7 @@ function wrapBetaChatCompletion(completion) {
       }
     });
     const startTime = getCurrentUnixTimestamp();
-    const ret = await completion(params);
+    const ret = completion(params);
     let first2 = true;
     ret.on("chunk", (_chunk) => {
       if (first2) {
@@ -9629,10 +9780,13 @@ function wrapBetaChatCompletion(completion) {
   };
 }
 function wrapChatCompletion(completion) {
-  return async (params) => {
+  return async (params, options) => {
     const { messages, ...rest } = params;
     const span = startSpan({
       name: "OpenAI Chat Completion",
+      spanAttributes: {
+        type: SpanTypeAttribute.LLM
+      },
       event: {
         input: messages,
         metadata: {
@@ -9642,11 +9796,14 @@ function wrapChatCompletion(completion) {
     });
     if (params.stream) {
       const startTime = getCurrentUnixTimestamp();
-      const ret = await completion(params);
+      const ret = await completion(params, options);
       return new WrapperStream(span, startTime, ret);
     } else {
       try {
-        const ret = await completion(params);
+        const ret = await completion(
+          params,
+          options
+        );
         const { messages: messages2, ...rest2 } = params;
         span.log({
           input: messages2,
@@ -9667,6 +9824,37 @@ function wrapChatCompletion(completion) {
     }
   };
 }
+function wrapEmbeddings(create) {
+  return async (params, options) => {
+    const { input, ...rest } = params;
+    return traced(
+      async (span) => {
+        const result = await create(params, options);
+        const output = result.data[0];
+        span.log({
+          output,
+          metrics: {
+            tokens: result.usage?.total_tokens,
+            prompt_tokens: result.usage?.prompt_tokens
+          }
+        });
+        return result;
+      },
+      {
+        name: "OpenAI Embedding",
+        spanAttributes: {
+          type: SpanTypeAttribute.LLM
+        },
+        event: {
+          input,
+          metadata: {
+            ...rest
+          }
+        }
+      }
+    );
+  };
+}
 var WrapperStream = class {
   constructor(span, startTime, iter) {
     this.span = span;

package/dist/logger.d.ts

@@ -1,5 +1,5 @@
 /// <reference lib="dom" />
-import { IS_MERGE_FIELD } from "@braintrust/core";
+import { IS_MERGE_FIELD, PARENT_ID_FIELD, Source, AUDIT_SOURCE_FIELD, AUDIT_METADATA_FIELD } from "@braintrust/core";
 import { IsoAsyncLocalStorage } from "./isomorph";
 export type Metadata = Record<string, unknown>;
 export type SetCurrentArg = {
@@ -10,6 +10,7 @@ export type StartSpanArgs = {
     name?: string;
     spanAttributes?: Record<any, any>;
     startTime?: number;
+    parentId?: string;
     event?: StartSpanEventArgs;
 };
 export type EndSpanArgs = {
@@ -41,6 +42,12 @@ export interface Span {
      * @param event: Data to be logged. See `Experiment.log` for full details.
      */
     log(event: ExperimentLogPartialArgs): void;
+    /**
+     * Add feedback to the current span. Unlike `Experiment.logFeedback` and `Logger.logFeedback`, this method does not accept an id parameter, because it logs feedback to the current span.
+     *
+     * @param event: Data to be logged. See `Experiment.logFeedback` for full details.
+     */
+    logFeedback(event: Omit<LogFeedbackFullArgs, "id">): void;
     /**
      * Create a new span and run the provided callback. This is useful if you want to log more detailed trace information beyond the scope of a single log event. Data logged over several calls to `Span.log` will be merged into one logical row.
      *
@@ -51,6 +58,7 @@ export interface Span {
      * @param args.span_attributes Optional additional attributes to attach to the span, such as a type name.
      * @param args.start_time Optional start time of the span, as a timestamp in seconds.
      * @param args.setCurrent If true (the default), the span will be marked as the currently-active span for the duration of the callback.
+     * @param args.parentId Optional id of the parent span. If not provided, the current span will be used (depending on context). This is useful for adding spans to an existing trace.
      * @param args.event Data to be logged. See `Experiment.log` for full details.
      * @Returns The result of running `callback`.
      */
@@ -88,6 +96,7 @@ export declare class NoopSpan implements Span {
     kind: "span";
     constructor();
     log(_: ExperimentLogPartialArgs): void;
+    logFeedback(event: Omit<LogFeedbackFullArgs, "id">): void;
     traced<R>(callback: (span: Span) => R, _1: StartSpanArgs & SetCurrentArg): R;
     startSpan(_1?: StartSpanArgs): this;
     end(args?: EndSpanArgs): number;
@@ -183,12 +192,25 @@ export declare class Logger<IsAsyncFlush extends boolean> {
      * See `Span.traced` for full details.
      */
     traced<R>(callback: (span: Span) => R, args?: StartSpanArgs & SetCurrentArg): PromiseUnless<IsAsyncFlush, R>;
+    private lazyParentIds;
     /**
      * Lower-level alternative to `traced`, which does not automatically end the span or mark it as current.
      *
      * See `traced` for full details.
      */
     startSpan(args?: StartSpanArgs): Span;
+    /**
+     * Log feedback to an event. Feedback is used to save feedback scores, set an expected value, or add a comment.
+     *
+     * @param event
+     * @param event.id The id of the event to log feedback for. This is the `id` returned by `log` or accessible as the `id` field of a span.
+     * @param event.scores (Optional) a dictionary of numeric values (between 0 and 1) to log. These scores will be merged into the existing scores for the event.
+     * @param event.expected (Optional) the ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to determine if your `output` value is correct or not.
+     * @param event.comment (Optional) an optional comment string to log about the event.
+     * @param event.metadata (Optional) a dictionary with additional data about the feedback. If you have a `user_id`, you can log it here and access it in the Braintrust UI.
+     * @param event.source (Optional) the source of the feedback. Must be one of "external" (default), "app", or "api".
+     */
+    logFeedback(event: LogFeedbackFullArgs): void;
     flush(): Promise<void>;
     get asyncFlush(): IsAsyncFlush | undefined;
 }
@@ -211,10 +233,25 @@ export type OtherExperimentLogFields = {
 };
 export type ExperimentLogPartialArgs = Partial<OtherExperimentLogFields> & Partial<InputField | InputsField>;
 export type ExperimentLogFullArgs = Partial<Omit<OtherExperimentLogFields, "scores">> & Required<Pick<OtherExperimentLogFields, "scores">> & Partial<InputField | InputsField> & Partial<IdField>;
+export type LogFeedbackFullArgs = IdField & Partial<Omit<OtherExperimentLogFields, "output" | "metrics" | "datasetRecordId"> & {
+    comment: string;
+    source: Source;
+}>;
+export type LogCommentFullArgs = IdField & {
+    created: string;
+    origin: {
+        id: string;
+    };
+    comment: {
+        text: string;
+    };
+    [AUDIT_SOURCE_FIELD]: Source;
+    [AUDIT_METADATA_FIELD]?: Record<string, unknown>;
+} & Omit<ParentExperimentIds | ParentProjectLogIds, "kind">;
 type ExperimentEvent = Partial<InputField> & Partial<OtherExperimentLogFields> & {
     id: string;
-    span_id: string;
-    root_span_id: string;
+    span_id?: string;
+    root_span_id?: string;
     project_id: string;
     experiment_id: string;
     [IS_MERGE_FIELD]: boolean;
@@ -222,6 +259,9 @@ type ExperimentEvent = Partial<InputField> & Partial<OtherExperimentLogFields> &
     created: string;
     span_parents: string[];
     span_attributes: Record<string, unknown>;
+    [PARENT_ID_FIELD]: string;
+    [AUDIT_SOURCE_FIELD]: Source;
+    [AUDIT_METADATA_FIELD]?: Record<string, unknown>;
 }>;
 interface DatasetEvent {
     inputs?: unknown;
@@ -236,7 +276,18 @@ type LoggingEvent = Omit<ExperimentEvent, "experiment_id"> & {
     org_id: string;
     log_id: "g";
 };
-type BackgroundLogEvent = ExperimentEvent | DatasetEvent | LoggingEvent;
+export type CommentEvent = IdField & {
+    created: string;
+    origin: {
+        id: string;
+    };
+    comment: {
+        text: string;
+    };
+    [AUDIT_SOURCE_FIELD]: Source;
+    [AUDIT_METADATA_FIELD]?: Record<string, unknown>;
+} & Omit<ParentExperimentIds | ParentProjectLogIds, "kind">;
+type BackgroundLogEvent = ExperimentEvent | DatasetEvent | LoggingEvent | CommentEvent;
 export interface DatasetRecord {
     id: string;
     input: any;
@@ -467,6 +518,7 @@ export declare class Experiment {
      * See `Span.traced` for full details.
      */
     traced<R>(callback: (span: Span) => R, args?: StartSpanArgs & SetCurrentArg): R;
+    private lazyParentIds;
     /**
      * Lower-level alternative to `traced`, which does not automatically end the span or mark it as current.
      *
@@ -485,6 +537,18 @@ export declare class Experiment {
         readonly summarizeScores?: boolean;
         readonly comparisonExperimentId?: string;
     }): Promise<ExperimentSummary>;
+    /**
+     * Log feedback to an event in the experiment. Feedback is used to save feedback scores, set an expected value, or add a comment.
+     *
+     * @param event
+     * @param event.id The id of the event to log feedback for. This is the `id` returned by `log` or accessible as the `id` field of a span.
+     * @param event.scores (Optional) a dictionary of numeric values (between 0 and 1) to log. These scores will be merged into the existing scores for the event.
+     * @param event.expected (Optional) the ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to determine if your `output` value is correct or not.
+     * @param event.comment (Optional) an optional comment string to log about the event.
+     * @param event.metadata (Optional) a dictionary with additional data about the feedback. If you have a `user_id`, you can log it here and access it in the Braintrust UI.
+     * @param event.source (Optional) the source of the feedback. Must be one of "external" (default), "app", or "api".
+     */
+    logFeedback(event: LogFeedbackFullArgs): void;
     /**
      * Flush any pending rows to the server.
      */
@@ -521,17 +585,21 @@ export declare class SpanImpl implements Span {
     constructor(args: {
         parentIds: Promise<ParentExperimentIds | ParentProjectLogIds>;
         bgLogger: BackgroundLogger;
+    } & Omit<StartSpanArgs, "parentId"> & ({
         parentSpanInfo?: {
             span_id: string;
             root_span_id: string;
         };
-    } & StartSpanArgs);
+    } | {
+        parentId?: string;
+    }));
     get id(): string;
     get span_id(): string;
     get root_span_id(): string;
     log(event: ExperimentLogPartialArgs): void;
+    logFeedback(event: Omit<LogFeedbackFullArgs, "id">): void;
     traced<R>(callback: (span: Span) => R, args?: StartSpanArgs & SetCurrentArg): R;
-    startSpan(args?: StartSpanArgs): Span;
+    startSpan(args?: Omit<StartSpanArgs, "parent_id">): Span;
     end(args?: EndSpanArgs): number;
     close(args?: EndSpanArgs): number;
 }

package/dist/oai.d.ts

@@ -4,14 +4,19 @@ interface BetaLike {
             stream: any;
         };
     };
+    embeddings: any;
 }
 interface ChatLike {
     completions: any;
 }
 interface OpenAILike {
     chat: ChatLike;
+    embeddings: any;
     beta?: BetaLike;
 }
+declare global {
+    var __inherited_braintrust_wrap_openai: ((openai: any) => any) | undefined;
+}
 /**
  * Wrap an `OpenAI` object (created with `new OpenAI(...)`) to add tracing. If Braintrust is
  * not configured, this is a no-op