braintrust 0.0.113 → 0.0.115
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser.js +539 -288
- package/dist/cli.js +672 -346
- package/dist/framework.d.ts +56 -23
- package/dist/index.d.ts +8 -3
- package/dist/index.js +691 -367
- package/dist/logger.d.ts +61 -33
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +2 -2
package/dist/browser.js
CHANGED
|
@@ -1670,6 +1670,88 @@ function makeLegacyEvent(e) {
|
|
|
1670
1670
|
}
|
|
1671
1671
|
return event;
|
|
1672
1672
|
}
|
|
1673
|
+
var SpanParentObjectType = /* @__PURE__ */ ((SpanParentObjectType2) => {
|
|
1674
|
+
SpanParentObjectType2["EXPERIMENT"] = "experiment";
|
|
1675
|
+
SpanParentObjectType2["PROJECT_LOGS"] = "project_logs";
|
|
1676
|
+
return SpanParentObjectType2;
|
|
1677
|
+
})(SpanParentObjectType || {});
|
|
1678
|
+
var _OBJECT_TYPE_TO_PREFIX = {
|
|
1679
|
+
[
|
|
1680
|
+
"experiment"
|
|
1681
|
+
/* EXPERIMENT */
|
|
1682
|
+
]: "ex",
|
|
1683
|
+
[
|
|
1684
|
+
"project_logs"
|
|
1685
|
+
/* PROJECT_LOGS */
|
|
1686
|
+
]: "pl"
|
|
1687
|
+
};
|
|
1688
|
+
var _PREFIX_TO_OBJECT_TYPE = Object.fromEntries(
|
|
1689
|
+
Object.entries(_OBJECT_TYPE_TO_PREFIX).map(([k, v]) => [v, k])
|
|
1690
|
+
);
|
|
1691
|
+
var _SEP = ":";
|
|
1692
|
+
var SpanParentComponents = class _SpanParentComponents {
|
|
1693
|
+
constructor(args) {
|
|
1694
|
+
this.objectType = args.objectType;
|
|
1695
|
+
this.objectId = args.objectId;
|
|
1696
|
+
this.rowId = args.rowId;
|
|
1697
|
+
if (!(typeof this.objectType === "string")) {
|
|
1698
|
+
throw new Error("objectType must be a string");
|
|
1699
|
+
}
|
|
1700
|
+
if (!(typeof this.objectId === "string")) {
|
|
1701
|
+
throw new Error("objectId must be a string");
|
|
1702
|
+
}
|
|
1703
|
+
if (!(typeof this.rowId === "string")) {
|
|
1704
|
+
throw new Error("rowId must be a string");
|
|
1705
|
+
}
|
|
1706
|
+
const objectTypePrefix = _OBJECT_TYPE_TO_PREFIX[this.objectType];
|
|
1707
|
+
if (objectTypePrefix.includes(_SEP)) {
|
|
1708
|
+
throw new Error(
|
|
1709
|
+
`objectType prefix ${objectTypePrefix} may not contain separator character ${_SEP}`
|
|
1710
|
+
);
|
|
1711
|
+
}
|
|
1712
|
+
if (this.objectId.includes(_SEP)) {
|
|
1713
|
+
throw new Error(
|
|
1714
|
+
`objectId ${this.objectId} may not contain separator character ${_SEP}`
|
|
1715
|
+
);
|
|
1716
|
+
}
|
|
1717
|
+
}
|
|
1718
|
+
toStr() {
|
|
1719
|
+
return [
|
|
1720
|
+
_OBJECT_TYPE_TO_PREFIX[this.objectType],
|
|
1721
|
+
this.objectId,
|
|
1722
|
+
this.rowId
|
|
1723
|
+
].join(_SEP);
|
|
1724
|
+
}
|
|
1725
|
+
static fromStr(s) {
|
|
1726
|
+
const items = s.split(_SEP);
|
|
1727
|
+
if (items.length < 3) {
|
|
1728
|
+
throw new Error(
|
|
1729
|
+
`Serialized parent components string must have at least three components. Provided string ${s} has only ${items.length}`
|
|
1730
|
+
);
|
|
1731
|
+
}
|
|
1732
|
+
return new _SpanParentComponents({
|
|
1733
|
+
objectType: _PREFIX_TO_OBJECT_TYPE[items[0]],
|
|
1734
|
+
objectId: items[1],
|
|
1735
|
+
rowId: items.slice(2).join(_SEP)
|
|
1736
|
+
});
|
|
1737
|
+
}
|
|
1738
|
+
asDict() {
|
|
1739
|
+
const out = (() => {
|
|
1740
|
+
switch (this.objectType) {
|
|
1741
|
+
case "experiment":
|
|
1742
|
+
return { experiment_id: this.objectId };
|
|
1743
|
+
case "project_logs":
|
|
1744
|
+
return { project_id: this.objectId, log_id: "g" };
|
|
1745
|
+
default:
|
|
1746
|
+
throw new Error("Impossible");
|
|
1747
|
+
}
|
|
1748
|
+
})();
|
|
1749
|
+
if (this.rowId) {
|
|
1750
|
+
out[PARENT_ID_FIELD] = this.rowId;
|
|
1751
|
+
}
|
|
1752
|
+
return out;
|
|
1753
|
+
}
|
|
1754
|
+
};
|
|
1673
1755
|
var SpanTypeAttribute = /* @__PURE__ */ ((SpanTypeAttribute22) => {
|
|
1674
1756
|
SpanTypeAttribute22["LLM"] = "llm";
|
|
1675
1757
|
SpanTypeAttribute22["SCORE"] = "score";
|
|
@@ -5482,62 +5564,66 @@ var modeToTypes = {
|
|
|
5482
5564
|
}
|
|
5483
5565
|
};
|
|
5484
5566
|
var customTypes = modeToTypes[mode];
|
|
5485
|
-
var chatCompletionSystemMessageParamSchema = z.
|
|
5486
|
-
content: z.string(),
|
|
5567
|
+
var chatCompletionSystemMessageParamSchema = z.strictObject({
|
|
5568
|
+
content: z.string().default(""),
|
|
5487
5569
|
role: z.literal("system"),
|
|
5488
5570
|
name: z.string().optional()
|
|
5489
5571
|
});
|
|
5490
|
-
var chatCompletionContentPartTextSchema = z.
|
|
5491
|
-
text: z.string(),
|
|
5572
|
+
var chatCompletionContentPartTextSchema = z.strictObject({
|
|
5573
|
+
text: z.string().default(""),
|
|
5492
5574
|
type: z.literal("text")
|
|
5493
5575
|
});
|
|
5494
|
-
var
|
|
5576
|
+
var imageURLSchema = z.strictObject({
|
|
5577
|
+
url: z.string(),
|
|
5578
|
+
detail: z.union([z.literal("auto"), z.literal("low"), z.literal("high")]).optional()
|
|
5579
|
+
});
|
|
5580
|
+
var chatCompletionContentPartImageSchema = z.strictObject({
|
|
5581
|
+
image_url: imageURLSchema,
|
|
5582
|
+
type: z.literal("image_url")
|
|
5583
|
+
});
|
|
5584
|
+
var chatCompletionContentPartSchema = z.union([
|
|
5585
|
+
chatCompletionContentPartTextSchema,
|
|
5586
|
+
chatCompletionContentPartImageSchema
|
|
5587
|
+
]);
|
|
5588
|
+
var chatCompletionContentSchema = z.union([
|
|
5589
|
+
z.string().default(""),
|
|
5590
|
+
z.array(chatCompletionContentPartSchema)
|
|
5591
|
+
]);
|
|
5592
|
+
var chatCompletionUserMessageParamSchema = z.strictObject({
|
|
5593
|
+
content: chatCompletionContentSchema,
|
|
5594
|
+
role: z.literal("user"),
|
|
5595
|
+
name: z.string().optional()
|
|
5596
|
+
});
|
|
5597
|
+
var functionCallSchema = z.strictObject({
|
|
5495
5598
|
arguments: z.string(),
|
|
5496
5599
|
name: z.string()
|
|
5497
5600
|
});
|
|
5498
|
-
var functionSchema = z.
|
|
5601
|
+
var functionSchema = z.strictObject({
|
|
5499
5602
|
arguments: z.string(),
|
|
5500
5603
|
name: z.string()
|
|
5501
5604
|
});
|
|
5502
|
-
var
|
|
5503
|
-
|
|
5504
|
-
detail: z.union([z.literal("auto"), z.literal("low"), z.literal("high")]).optional()
|
|
5505
|
-
});
|
|
5506
|
-
var chatCompletionToolMessageParamSchema = z.object({
|
|
5507
|
-
content: z.string(),
|
|
5605
|
+
var chatCompletionToolMessageParamSchema = z.strictObject({
|
|
5606
|
+
content: z.string().default(""),
|
|
5508
5607
|
role: z.literal("tool"),
|
|
5509
5608
|
tool_call_id: z.string()
|
|
5510
5609
|
});
|
|
5511
|
-
var chatCompletionFunctionMessageParamSchema = z.
|
|
5512
|
-
content: z.string().
|
|
5610
|
+
var chatCompletionFunctionMessageParamSchema = z.strictObject({
|
|
5611
|
+
content: z.string().default(""),
|
|
5513
5612
|
name: z.string(),
|
|
5514
5613
|
role: z.literal("function")
|
|
5515
5614
|
});
|
|
5516
|
-
var
|
|
5517
|
-
image_url: imageURLSchema,
|
|
5518
|
-
type: z.literal("image_url")
|
|
5519
|
-
});
|
|
5520
|
-
var chatCompletionMessageToolCallSchema = z.object({
|
|
5615
|
+
var chatCompletionMessageToolCallSchema = z.strictObject({
|
|
5521
5616
|
id: z.string(),
|
|
5522
5617
|
function: functionSchema,
|
|
5523
5618
|
type: z.literal("function")
|
|
5524
5619
|
});
|
|
5525
|
-
var
|
|
5526
|
-
chatCompletionContentPartTextSchema,
|
|
5527
|
-
chatCompletionContentPartImageSchema
|
|
5528
|
-
]);
|
|
5529
|
-
var chatCompletionAssistantMessageParamSchema = z.object({
|
|
5620
|
+
var chatCompletionAssistantMessageParamSchema = z.strictObject({
|
|
5530
5621
|
role: z.literal("assistant"),
|
|
5531
|
-
content: z.string().
|
|
5622
|
+
content: z.string().nullish(),
|
|
5532
5623
|
function_call: functionCallSchema.optional(),
|
|
5533
5624
|
name: z.string().optional(),
|
|
5534
5625
|
tool_calls: z.array(chatCompletionMessageToolCallSchema).optional()
|
|
5535
5626
|
});
|
|
5536
|
-
var chatCompletionUserMessageParamSchema = z.object({
|
|
5537
|
-
content: z.union([z.string(), z.array(chatCompletionContentPartSchema)]),
|
|
5538
|
-
role: z.literal("user"),
|
|
5539
|
-
name: z.string().optional()
|
|
5540
|
-
});
|
|
5541
5627
|
var chatCompletionMessageParamSchema = z.union([
|
|
5542
5628
|
chatCompletionSystemMessageParamSchema,
|
|
5543
5629
|
chatCompletionUserMessageParamSchema,
|
|
@@ -5546,12 +5632,12 @@ var chatCompletionMessageParamSchema = z.union([
|
|
|
5546
5632
|
chatCompletionFunctionMessageParamSchema
|
|
5547
5633
|
]);
|
|
5548
5634
|
var functionParametersSchema = z.record(z.unknown());
|
|
5549
|
-
var functionDefinitionSchema = z.
|
|
5635
|
+
var functionDefinitionSchema = z.strictObject({
|
|
5550
5636
|
name: z.string(),
|
|
5551
5637
|
description: z.string().optional(),
|
|
5552
5638
|
parameters: functionParametersSchema.optional()
|
|
5553
5639
|
});
|
|
5554
|
-
var chatCompletionToolSchema = z.
|
|
5640
|
+
var chatCompletionToolSchema = z.strictObject({
|
|
5555
5641
|
function: functionDefinitionSchema,
|
|
5556
5642
|
type: z.literal("function")
|
|
5557
5643
|
});
|
|
@@ -5565,69 +5651,71 @@ var messageRoleSchema = z.enum([
|
|
|
5565
5651
|
"model"
|
|
5566
5652
|
]);
|
|
5567
5653
|
var promptBlockDataSchema = z.union([
|
|
5568
|
-
z.
|
|
5654
|
+
z.strictObject({
|
|
5569
5655
|
type: z.literal("completion"),
|
|
5570
5656
|
content: z.string()
|
|
5571
5657
|
}),
|
|
5572
|
-
z.
|
|
5658
|
+
z.strictObject({
|
|
5573
5659
|
type: z.literal("chat"),
|
|
5574
5660
|
messages: z.array(chatCompletionMessageParamSchema),
|
|
5575
5661
|
tools: z.string().optional()
|
|
5576
5662
|
})
|
|
5577
5663
|
]);
|
|
5578
|
-
var braintrustModelParamsSchema = z.
|
|
5664
|
+
var braintrustModelParamsSchema = z.strictObject({
|
|
5579
5665
|
use_cache: z.boolean().optional()
|
|
5580
5666
|
});
|
|
5581
5667
|
var BRAINTRUST_PARAMS = Object.keys(braintrustModelParamsSchema.shape);
|
|
5582
|
-
var openAIModelParamsSchema = z.
|
|
5583
|
-
temperature: z.number(),
|
|
5668
|
+
var openAIModelParamsSchema = z.strictObject({
|
|
5669
|
+
temperature: z.number().optional(),
|
|
5584
5670
|
top_p: z.number().optional(),
|
|
5585
5671
|
max_tokens: z.number().optional(),
|
|
5586
5672
|
frequency_penalty: z.number().optional(),
|
|
5587
5673
|
presence_penalty: z.number().optional(),
|
|
5588
|
-
response_format: z.union([
|
|
5674
|
+
response_format: z.union([
|
|
5675
|
+
z.literal(null),
|
|
5676
|
+
z.strictObject({ type: z.literal("json_object") })
|
|
5677
|
+
]).optional(),
|
|
5589
5678
|
tool_choice: z.union([
|
|
5590
5679
|
z.literal("auto"),
|
|
5591
5680
|
z.literal("none"),
|
|
5592
|
-
z.
|
|
5681
|
+
z.strictObject({
|
|
5593
5682
|
type: z.literal("function"),
|
|
5594
|
-
function: z.
|
|
5683
|
+
function: z.strictObject({ name: z.string() })
|
|
5595
5684
|
})
|
|
5596
5685
|
]).optional()
|
|
5597
5686
|
});
|
|
5598
|
-
var anthropicModelParamsSchema = z.
|
|
5687
|
+
var anthropicModelParamsSchema = z.strictObject({
|
|
5599
5688
|
max_tokens: z.number(),
|
|
5600
5689
|
temperature: z.number(),
|
|
5601
5690
|
top_p: z.number().optional(),
|
|
5602
5691
|
top_k: z.number().optional(),
|
|
5603
5692
|
max_tokens_to_sample: z.number().optional().describe("This is a legacy parameter that should not be used.")
|
|
5604
5693
|
});
|
|
5605
|
-
var googleModelParamsSchema = z.
|
|
5694
|
+
var googleModelParamsSchema = z.strictObject({
|
|
5606
5695
|
temperature: z.number(),
|
|
5607
5696
|
maxOutputTokens: z.number().optional(),
|
|
5608
5697
|
topP: z.number().optional(),
|
|
5609
5698
|
topK: z.number().optional()
|
|
5610
5699
|
});
|
|
5611
|
-
var jsCompletionParamsSchema = z.
|
|
5612
|
-
var modelParamsSchema =
|
|
5613
|
-
|
|
5614
|
-
|
|
5615
|
-
|
|
5616
|
-
|
|
5617
|
-
|
|
5618
|
-
|
|
5619
|
-
|
|
5620
|
-
var anyModelParamsSchema = openAIModelParamsSchema.and(anthropicModelParamsSchema).and(googleModelParamsSchema).and(braintrustModelParamsSchema);
|
|
5621
|
-
var promptOptionsSchema = z.object({
|
|
5700
|
+
var jsCompletionParamsSchema = z.strictObject({});
|
|
5701
|
+
var modelParamsSchema = z.union([
|
|
5702
|
+
braintrustModelParamsSchema.merge(openAIModelParamsSchema),
|
|
5703
|
+
braintrustModelParamsSchema.merge(anthropicModelParamsSchema),
|
|
5704
|
+
braintrustModelParamsSchema.merge(googleModelParamsSchema),
|
|
5705
|
+
braintrustModelParamsSchema.merge(jsCompletionParamsSchema)
|
|
5706
|
+
]);
|
|
5707
|
+
var anyModelParamsSchema = openAIModelParamsSchema.merge(anthropicModelParamsSchema).merge(googleModelParamsSchema).merge(braintrustModelParamsSchema);
|
|
5708
|
+
var promptOptionsSchema = z.strictObject({
|
|
5622
5709
|
model: z.string().optional(),
|
|
5623
5710
|
params: modelParamsSchema.optional(),
|
|
5624
5711
|
position: z.string().optional()
|
|
5625
5712
|
});
|
|
5626
|
-
var promptDataSchema = z.
|
|
5713
|
+
var promptDataSchema = z.strictObject({
|
|
5627
5714
|
prompt: promptBlockDataSchema.nullish(),
|
|
5628
5715
|
options: promptOptionsSchema.nullish(),
|
|
5629
|
-
origin: z.
|
|
5716
|
+
origin: z.strictObject({
|
|
5630
5717
|
prompt_id: z.string().optional(),
|
|
5718
|
+
project_id: z.string().optional(),
|
|
5631
5719
|
prompt_version: z.string().optional()
|
|
5632
5720
|
}).nullish()
|
|
5633
5721
|
}).openapi("PromptData");
|
|
@@ -5637,7 +5725,7 @@ function generateBaseTableSchema(objectName, opts) {
|
|
|
5637
5725
|
if (opts == null ? void 0 : opts.uniqueName) {
|
|
5638
5726
|
nameDescription += `. Within a project, ${objectName} names are unique`;
|
|
5639
5727
|
}
|
|
5640
|
-
return z.
|
|
5728
|
+
return z.strictObject({
|
|
5641
5729
|
id: z.string().uuid().describe(`Unique identifier for the ${objectName}`),
|
|
5642
5730
|
project_id: z.string().uuid().describe(
|
|
5643
5731
|
`Unique identifier for the project that the ${objectName} belongs under`
|
|
@@ -5653,7 +5741,7 @@ function generateBaseTableSchema(objectName, opts) {
|
|
|
5653
5741
|
});
|
|
5654
5742
|
}
|
|
5655
5743
|
var userBaseSchema = generateBaseTableSchema("user");
|
|
5656
|
-
var userSchema = z.
|
|
5744
|
+
var userSchema = z.strictObject({
|
|
5657
5745
|
id: userBaseSchema.shape.id,
|
|
5658
5746
|
auth_id: z.string().uuid().nullish().describe("Internal authentication token used to identify the user"),
|
|
5659
5747
|
given_name: z.string().nullish().describe("Given name of the user"),
|
|
@@ -5661,29 +5749,29 @@ var userSchema = z.object({
|
|
|
5661
5749
|
email: z.string().nullish().describe("The user's email"),
|
|
5662
5750
|
avatar_url: z.string().nullish().describe("URL of the user's Avatar image"),
|
|
5663
5751
|
created: userBaseSchema.shape.created
|
|
5664
|
-
}).
|
|
5752
|
+
}).openapi("User");
|
|
5665
5753
|
var organizationBaseSchema = generateBaseTableSchema("organization");
|
|
5666
|
-
var organizationSchema = z.
|
|
5754
|
+
var organizationSchema = z.strictObject({
|
|
5667
5755
|
id: organizationBaseSchema.shape.id,
|
|
5668
5756
|
name: organizationBaseSchema.shape.name.nullish(),
|
|
5669
5757
|
api_url: z.string().nullish(),
|
|
5670
5758
|
created: organizationBaseSchema.shape.created
|
|
5671
|
-
}).
|
|
5672
|
-
var memberSchema = z.
|
|
5759
|
+
}).openapi("Organization");
|
|
5760
|
+
var memberSchema = z.strictObject({
|
|
5673
5761
|
org_id: organizationSchema.shape.id,
|
|
5674
5762
|
user_id: userSchema.shape.id
|
|
5675
|
-
}).
|
|
5676
|
-
var meSchema = z.
|
|
5763
|
+
}).openapi("Member");
|
|
5764
|
+
var meSchema = z.strictObject({
|
|
5677
5765
|
id: userSchema.shape.id,
|
|
5678
5766
|
// By filtering by auth_id equality, we will ensure this is not-null.
|
|
5679
5767
|
auth_id: userSchema.shape.auth_id.unwrap().unwrap(),
|
|
5680
|
-
organizations: z.
|
|
5768
|
+
organizations: z.strictObject({
|
|
5681
5769
|
id: memberSchema.shape.org_id,
|
|
5682
5770
|
name: organizationSchema.shape.name
|
|
5683
5771
|
}).array()
|
|
5684
|
-
}).
|
|
5772
|
+
}).openapi("Me");
|
|
5685
5773
|
var apiKeyBaseSchema = generateBaseTableSchema("api key");
|
|
5686
|
-
var apiKeySchema = z.
|
|
5774
|
+
var apiKeySchema = z.strictObject({
|
|
5687
5775
|
id: apiKeyBaseSchema.shape.id,
|
|
5688
5776
|
created: apiKeyBaseSchema.shape.created,
|
|
5689
5777
|
key_hash: z.string(),
|
|
@@ -5691,9 +5779,9 @@ var apiKeySchema = z.object({
|
|
|
5691
5779
|
preview_name: z.string(),
|
|
5692
5780
|
user_id: userSchema.shape.id.nullish(),
|
|
5693
5781
|
org_id: organizationSchema.shape.id.nullish()
|
|
5694
|
-
}).
|
|
5782
|
+
}).openapi("ApiKey");
|
|
5695
5783
|
var projectBaseSchema = generateBaseTableSchema("project");
|
|
5696
|
-
var projectSchema = z.
|
|
5784
|
+
var projectSchema = z.strictObject({
|
|
5697
5785
|
id: projectBaseSchema.shape.id,
|
|
5698
5786
|
org_id: z.string().uuid().describe(
|
|
5699
5787
|
"Unique id for the organization that the project belongs under"
|
|
@@ -5702,11 +5790,11 @@ var projectSchema = z.object({
|
|
|
5702
5790
|
created: projectBaseSchema.shape.created,
|
|
5703
5791
|
deleted_at: projectBaseSchema.shape.deleted_at,
|
|
5704
5792
|
user_id: projectBaseSchema.shape.user_id
|
|
5705
|
-
}).
|
|
5793
|
+
}).openapi("Project");
|
|
5706
5794
|
var datasetBaseSchema = generateBaseTableSchema("dataset", {
|
|
5707
5795
|
uniqueName: true
|
|
5708
5796
|
});
|
|
5709
|
-
var datasetSchema = z.
|
|
5797
|
+
var datasetSchema = z.strictObject({
|
|
5710
5798
|
id: datasetBaseSchema.shape.id,
|
|
5711
5799
|
project_id: datasetBaseSchema.shape.project_id.nullish(),
|
|
5712
5800
|
name: datasetBaseSchema.shape.name,
|
|
@@ -5714,22 +5802,26 @@ var datasetSchema = z.object({
|
|
|
5714
5802
|
created: datasetBaseSchema.shape.created,
|
|
5715
5803
|
deleted_at: datasetBaseSchema.shape.deleted_at,
|
|
5716
5804
|
user_id: datasetBaseSchema.shape.user_id
|
|
5717
|
-
}).
|
|
5805
|
+
}).openapi("Dataset");
|
|
5718
5806
|
var promptBaseSchema = generateBaseTableSchema("prompt");
|
|
5719
|
-
var promptSchema = z.
|
|
5807
|
+
var promptSchema = z.strictObject({
|
|
5720
5808
|
id: promptBaseSchema.shape.id,
|
|
5721
5809
|
// This has to be copy/pasted because zod blows up when there are circular dependencies
|
|
5722
5810
|
_xact_id: z.string().describe(
|
|
5723
5811
|
`The transaction id of an event is unique to the network operation that processed the event insertion. Transaction ids are monotonically increasing over time and can be used to retrieve a versioned snapshot of the prompt (see the \`version\` parameter)`
|
|
5724
5812
|
),
|
|
5725
5813
|
project_id: promptBaseSchema.shape.project_id,
|
|
5814
|
+
log_id: z.literal("p").describe("A literal 'p' which identifies the object as a project prompt"),
|
|
5815
|
+
org_id: organizationSchema.shape.id,
|
|
5726
5816
|
name: promptBaseSchema.shape.name,
|
|
5727
5817
|
slug: z.string().describe("Unique identifier for the prompt"),
|
|
5728
5818
|
description: promptBaseSchema.shape.description,
|
|
5819
|
+
created: promptBaseSchema.shape.created,
|
|
5729
5820
|
prompt_data: promptDataSchema.nullish().describe("The prompt, model, and its parameters"),
|
|
5730
|
-
tags: z.array(z.string()).nullish().describe("A list of tags for the prompt")
|
|
5821
|
+
tags: z.array(z.string()).nullish().describe("A list of tags for the prompt"),
|
|
5822
|
+
metadata: promptBaseSchema.shape.metadata
|
|
5731
5823
|
});
|
|
5732
|
-
var repoInfoSchema = z.
|
|
5824
|
+
var repoInfoSchema = z.strictObject({
|
|
5733
5825
|
commit: z.string().nullish().describe("SHA of most recent commit"),
|
|
5734
5826
|
branch: z.string().nullish().describe("Name of the branch the most recent commit belongs to"),
|
|
5735
5827
|
tag: z.string().nullish().describe("Name of the tag on the most recent commit"),
|
|
@@ -5749,7 +5841,7 @@ var repoInfoSchema = z.object({
|
|
|
5749
5841
|
var experimentBaseSchema = generateBaseTableSchema("experiment", {
|
|
5750
5842
|
uniqueName: true
|
|
5751
5843
|
});
|
|
5752
|
-
var experimentSchema = z.
|
|
5844
|
+
var experimentSchema = z.strictObject({
|
|
5753
5845
|
id: experimentBaseSchema.shape.id,
|
|
5754
5846
|
project_id: experimentBaseSchema.shape.project_id,
|
|
5755
5847
|
name: experimentBaseSchema.shape.name,
|
|
@@ -5772,10 +5864,10 @@ var experimentSchema = z.object({
|
|
|
5772
5864
|
),
|
|
5773
5865
|
user_id: experimentBaseSchema.shape.user_id,
|
|
5774
5866
|
metadata: experimentBaseSchema.shape.metadata
|
|
5775
|
-
}).
|
|
5867
|
+
}).openapi("Experiment");
|
|
5776
5868
|
var appLimitSchema = z.number().int().nonnegative().describe("Limit the number of objects to return");
|
|
5777
5869
|
function generateBaseTableOpSchema(objectName) {
|
|
5778
|
-
return z.
|
|
5870
|
+
return z.strictObject({
|
|
5779
5871
|
org_name: z.string().nullish().describe(
|
|
5780
5872
|
`For nearly all users, this parameter should be unnecessary. But in the rare case that your API key belongs to multiple organizations, you may specify the name of the organization the ${objectName} belongs in.`
|
|
5781
5873
|
)
|
|
@@ -5794,14 +5886,14 @@ var endingBeforeSchema = z.string().uuid().describe(
|
|
|
5794
5886
|
].join("\n\n")
|
|
5795
5887
|
).openapi("EndingBefore");
|
|
5796
5888
|
var createProjectBaseSchema = generateBaseTableOpSchema("project");
|
|
5797
|
-
var createProjectSchema = z.
|
|
5889
|
+
var createProjectSchema = z.strictObject({
|
|
5798
5890
|
name: projectSchema.shape.name,
|
|
5799
5891
|
org_name: createProjectBaseSchema.shape.org_name
|
|
5800
|
-
}).
|
|
5801
|
-
var patchProjectSchema = z.
|
|
5892
|
+
}).openapi("CreateProject");
|
|
5893
|
+
var patchProjectSchema = z.strictObject({
|
|
5802
5894
|
name: projectSchema.shape.name.nullish()
|
|
5803
|
-
}).
|
|
5804
|
-
var createExperimentSchema = z.
|
|
5895
|
+
}).openapi("PatchProject");
|
|
5896
|
+
var createExperimentSchema = z.strictObject({
|
|
5805
5897
|
project_id: experimentSchema.shape.project_id,
|
|
5806
5898
|
name: experimentSchema.shape.name.nullish(),
|
|
5807
5899
|
description: experimentSchema.shape.description,
|
|
@@ -5811,21 +5903,28 @@ var createExperimentSchema = z.object({
|
|
|
5811
5903
|
dataset_version: experimentSchema.shape.dataset_version,
|
|
5812
5904
|
public: experimentSchema.shape.public.nullish(),
|
|
5813
5905
|
metadata: experimentSchema.shape.metadata
|
|
5814
|
-
}).
|
|
5815
|
-
var patchExperimentSchema = createExperimentSchema.omit({ project_id: true }).
|
|
5816
|
-
var createDatasetSchema = z.
|
|
5906
|
+
}).openapi("CreateExperiment");
|
|
5907
|
+
var patchExperimentSchema = createExperimentSchema.omit({ project_id: true }).openapi("PatchExperiment");
|
|
5908
|
+
var createDatasetSchema = z.strictObject({
|
|
5817
5909
|
project_id: datasetSchema.shape.project_id,
|
|
5818
5910
|
name: datasetSchema.shape.name,
|
|
5819
5911
|
description: datasetSchema.shape.description
|
|
5820
|
-
}).
|
|
5821
|
-
var patchDatasetSchema = createDatasetSchema.omit({ project_id: true }).
|
|
5822
|
-
var createPromptSchema = promptSchema.omit({
|
|
5823
|
-
|
|
5912
|
+
}).openapi("CreateDataset");
|
|
5913
|
+
var patchDatasetSchema = createDatasetSchema.omit({ project_id: true }).openapi("PatchDataset");
|
|
5914
|
+
var createPromptSchema = promptSchema.omit({
|
|
5915
|
+
id: true,
|
|
5916
|
+
_xact_id: true,
|
|
5917
|
+
org_id: true,
|
|
5918
|
+
log_id: true,
|
|
5919
|
+
created: true,
|
|
5920
|
+
metadata: true
|
|
5921
|
+
}).openapi("CreatePrompt");
|
|
5922
|
+
var patchPromptSchema = z.strictObject({
|
|
5824
5923
|
name: promptSchema.shape.name.nullish(),
|
|
5825
5924
|
description: promptSchema.shape.description.nullish(),
|
|
5826
5925
|
prompt_data: promptSchema.shape.prompt_data.nullish(),
|
|
5827
5926
|
tags: promptSchema.shape.tags.nullish()
|
|
5828
|
-
}).
|
|
5927
|
+
}).openapi("PatchPrompt");
|
|
5829
5928
|
function capitalize(s, sep) {
|
|
5830
5929
|
const items = sep ? s.split(sep) : [s];
|
|
5831
5930
|
return items.map((s2) => s2 ? s2.charAt(0).toUpperCase() + s2.slice(1) : s2).join(sep || "");
|
|
@@ -5849,7 +5948,7 @@ var SpanTypeAttribute2 = /* @__PURE__ */ ((SpanTypeAttribute22) => {
|
|
|
5849
5948
|
var auditSourcesSchema = z.enum(VALID_SOURCES2);
|
|
5850
5949
|
function generateBaseEventOpSchema(objectType2) {
|
|
5851
5950
|
const eventDescription = getEventObjectDescription(objectType2);
|
|
5852
|
-
return z.
|
|
5951
|
+
return z.strictObject({
|
|
5853
5952
|
id: z.string().describe(
|
|
5854
5953
|
`A unique identifier for the ${eventDescription} event. If you don't provide one, BrainTrust will generate one for you`
|
|
5855
5954
|
),
|
|
@@ -5865,7 +5964,7 @@ function generateBaseEventOpSchema(objectType2) {
|
|
|
5865
5964
|
metadata: z.record(customTypes.any).nullish().describe(
|
|
5866
5965
|
"A dictionary with additional data about the test example, model outputs, or just about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any JSON-serializable type, but its keys must be strings"
|
|
5867
5966
|
),
|
|
5868
|
-
metrics: z.
|
|
5967
|
+
metrics: z.strictObject({
|
|
5869
5968
|
start: z.number().nullish().describe(
|
|
5870
5969
|
`A unix timestamp recording when the section of code which produced the ${eventDescription} event started`
|
|
5871
5970
|
),
|
|
@@ -5875,7 +5974,7 @@ function generateBaseEventOpSchema(objectType2) {
|
|
|
5875
5974
|
}).catchall(customTypes.any).nullish().describe(
|
|
5876
5975
|
`Metrics are numerical measurements tracking the execution of the code that produced the ${eventDescription} event. Use "start" and "end" to track the time span over which the ${eventDescription} event was produced`
|
|
5877
5976
|
),
|
|
5878
|
-
context: z.
|
|
5977
|
+
context: z.strictObject({
|
|
5879
5978
|
caller_functionname: z.string().nullish().describe(
|
|
5880
5979
|
`The function in code which created the ${eventDescription} event`
|
|
5881
5980
|
),
|
|
@@ -5897,7 +5996,7 @@ function generateBaseEventOpSchema(objectType2) {
|
|
|
5897
5996
|
root_span_id: z.string().describe(
|
|
5898
5997
|
`The \`span_id\` of the root of the trace this ${eventDescription} event belongs to`
|
|
5899
5998
|
),
|
|
5900
|
-
span_attributes: z.
|
|
5999
|
+
span_attributes: z.strictObject({
|
|
5901
6000
|
name: z.string().nullish().describe("Name of the span, for display purposes only"),
|
|
5902
6001
|
type: z.nativeEnum(SpanTypeAttribute2).nullish().describe("Type of the span, for display purposes only")
|
|
5903
6002
|
}).catchall(customTypes.any).nullish().describe(
|
|
@@ -5911,7 +6010,7 @@ function generateBaseEventOpSchema(objectType2) {
|
|
|
5911
6010
|
function generateBaseEventFeedbackSchema(objectType2) {
|
|
5912
6011
|
const eventObjectType = getEventObjectType(objectType2);
|
|
5913
6012
|
const eventDescription = getEventObjectDescription(objectType2);
|
|
5914
|
-
return z.
|
|
6013
|
+
return z.strictObject({
|
|
5915
6014
|
id: z.string().describe(
|
|
5916
6015
|
`The id of the ${eventDescription} event to log feedback for. This is the row \`id\` returned by \`POST /v1/${eventObjectType}/{${objectType2}_id}/insert\``
|
|
5917
6016
|
),
|
|
@@ -5952,7 +6051,7 @@ var versionSchema = z.string().describe(
|
|
|
5952
6051
|
"The version id is essentially a filter on the latest event transaction id. You can use the `max_xact_id` returned by a past fetch as the version to reproduce that exact fetch."
|
|
5953
6052
|
].join("\n\n")
|
|
5954
6053
|
);
|
|
5955
|
-
var pathTypeFilterSchema = z.
|
|
6054
|
+
var pathTypeFilterSchema = z.strictObject({
|
|
5956
6055
|
type: z.literal("path_lookup").describe("Denotes the type of filter as a path-lookup filter"),
|
|
5957
6056
|
path: z.string().array().describe(
|
|
5958
6057
|
'List of fields describing the path to the value to be checked against. For instance, if you wish to filter on the value of `c` in `{"input": {"a": {"b": {"c": "hello"}}}}`, pass `path=["input", "a", "b", "c"]`'
|
|
@@ -5963,7 +6062,7 @@ var pathTypeFilterSchema = z.object({
|
|
|
5963
6062
|
}).describe(
|
|
5964
6063
|
'A path-lookup filter describes an equality comparison against a specific sub-field in the event row. For instance, if you wish to filter on the value of `c` in `{"input": {"a": {"b": {"c": "hello"}}}}`, pass `path=["input", "a", "b", "c"]` and `value="hello"`'
|
|
5965
6064
|
).openapi("PathLookupFilter");
|
|
5966
|
-
var sqlTypeFilterSchema = z.
|
|
6065
|
+
var sqlTypeFilterSchema = z.strictObject({
|
|
5967
6066
|
type: z.literal("sql_filter").describe("Denotes the type of filter as a sql-type filter"),
|
|
5968
6067
|
expr: z.string().describe(
|
|
5969
6068
|
`A SQL expression in [duckDB syntax](https://duckdb.org/docs/sql/expressions/overview). For instance, if you wish to fuzzy-match the value of \`c\` in \`{"input": {"a": {"b": {"c": "hello"}}}}\`, pass \`expr="input->'a'->'b'->>'c' LIKE '%el%'"\`.`
|
|
@@ -5977,24 +6076,24 @@ var allFetchFiltersSchema = z.union([pathTypeFilterSchema, sqlTypeFilterSchema])
|
|
|
5977
6076
|
var fetchFiltersSchema = pathTypeFilterSchema.array().describe(
|
|
5978
6077
|
"A list of filters on the events to fetch. Currently, only path-lookup type filters are supported, but we may add more in the future"
|
|
5979
6078
|
).openapi("FetchEventsFilters");
|
|
5980
|
-
var fetchEventsRequestSchema = z.
|
|
6079
|
+
var fetchEventsRequestSchema = z.strictObject({
|
|
5981
6080
|
limit: fetchLimitSchema.nullish(),
|
|
5982
6081
|
max_xact_id: maxXactIdSchema.nullish(),
|
|
5983
6082
|
max_root_span_id: maxRootSpanIdSchema.nullish(),
|
|
5984
6083
|
filters: fetchFiltersSchema.nullish(),
|
|
5985
6084
|
version: versionSchema.nullish()
|
|
5986
|
-
}).
|
|
6085
|
+
}).openapi("FetchEventsRequest");
|
|
5987
6086
|
function makeFetchEventsResponseSchema(objectType2, eventSchema) {
|
|
5988
6087
|
const eventName = capitalize(getEventObjectType(objectType2), "_").replace(
|
|
5989
6088
|
"_",
|
|
5990
6089
|
""
|
|
5991
6090
|
);
|
|
5992
|
-
return z.
|
|
6091
|
+
return z.strictObject({
|
|
5993
6092
|
events: eventSchema.array().describe("A list of fetched events")
|
|
5994
|
-
}).
|
|
6093
|
+
}).openapi(`Fetch${eventName}EventsResponse`);
|
|
5995
6094
|
}
|
|
5996
6095
|
var experimentEventBaseSchema = generateBaseEventOpSchema("experiment");
|
|
5997
|
-
var experimentEventSchema = z.
|
|
6096
|
+
var experimentEventSchema = z.strictObject({
|
|
5998
6097
|
id: experimentEventBaseSchema.shape.id,
|
|
5999
6098
|
dataset_record_id: z.string().nullish().describe(
|
|
6000
6099
|
"If the experiment is associated to a dataset, this is the event-level dataset id this experiment event is tied to"
|
|
@@ -6023,9 +6122,9 @@ var experimentEventSchema = z.object({
|
|
|
6023
6122
|
span_parents: experimentEventBaseSchema.shape.span_parents,
|
|
6024
6123
|
root_span_id: experimentEventBaseSchema.shape.root_span_id,
|
|
6025
6124
|
span_attributes: experimentEventBaseSchema.shape.span_attributes
|
|
6026
|
-
}).
|
|
6125
|
+
}).openapi("ExperimentEvent");
|
|
6027
6126
|
var datasetEventBaseSchema = generateBaseEventOpSchema("dataset");
|
|
6028
|
-
var datasetEventSchema = z.
|
|
6127
|
+
var datasetEventSchema = z.strictObject({
|
|
6029
6128
|
id: datasetEventBaseSchema.shape.id,
|
|
6030
6129
|
[TRANSACTION_ID_FIELD2]: datasetEventBaseSchema.shape[TRANSACTION_ID_FIELD2],
|
|
6031
6130
|
created: datasetEventBaseSchema.shape.created,
|
|
@@ -6041,9 +6140,9 @@ var datasetEventSchema = z.object({
|
|
|
6041
6140
|
tags: datasetEventBaseSchema.shape.tags,
|
|
6042
6141
|
span_id: datasetEventBaseSchema.shape.span_id,
|
|
6043
6142
|
root_span_id: datasetEventBaseSchema.shape.root_span_id
|
|
6044
|
-
}).
|
|
6143
|
+
}).openapi("DatasetEvent");
|
|
6045
6144
|
var projectLogsEventBaseSchema = generateBaseEventOpSchema("project");
|
|
6046
|
-
var projectLogsEventSchema = z.
|
|
6145
|
+
var projectLogsEventSchema = z.strictObject({
|
|
6047
6146
|
id: projectLogsEventBaseSchema.shape.id,
|
|
6048
6147
|
[TRANSACTION_ID_FIELD2]: projectLogsEventBaseSchema.shape[TRANSACTION_ID_FIELD2],
|
|
6049
6148
|
created: projectLogsEventBaseSchema.shape.created,
|
|
@@ -6070,12 +6169,12 @@ var projectLogsEventSchema = z.object({
|
|
|
6070
6169
|
span_parents: projectLogsEventBaseSchema.shape.span_parents,
|
|
6071
6170
|
root_span_id: projectLogsEventBaseSchema.shape.root_span_id,
|
|
6072
6171
|
span_attributes: projectLogsEventBaseSchema.shape.span_attributes
|
|
6073
|
-
}).
|
|
6172
|
+
}).openapi("ProjectLogsEvent");
|
|
6074
6173
|
var isMergeDescription = [
|
|
6075
6174
|
"The `_is_merge` field controls how the row is merged with any existing row with the same id in the DB. By default (or when set to `false`), the existing row is completely replaced by the new row. When set to `true`, the new row is deep-merged into the existing row",
|
|
6076
6175
|
'For example, say there is an existing row in the DB `{"id": "foo", "input": {"a": 5, "b": 10}}`. If we merge a new row as `{"_is_merge": true, "id": "foo", "input": {"b": 11, "c": 20}}`, the new row will be `{"id": "foo", "input": {"a": 5, "b": 11, "c": 20}}`. If we replace the new row as `{"id": "foo", "input": {"b": 11, "c": 20}}`, the new row will be `{"id": "foo", "input": {"b": 11, "c": 20}}`'
|
|
6077
6176
|
].join("\n\n");
|
|
6078
|
-
var mergeEventSchema = z.
|
|
6177
|
+
var mergeEventSchema = z.strictObject({
|
|
6079
6178
|
[IS_MERGE_FIELD2]: customTypes.literalTrue.describe(isMergeDescription),
|
|
6080
6179
|
[MERGE_PATHS_FIELD2]: z.string().array().array().nullish().describe(
|
|
6081
6180
|
[
|
|
@@ -6084,7 +6183,7 @@ var mergeEventSchema = z.object({
|
|
|
6084
6183
|
].join("\n\n")
|
|
6085
6184
|
)
|
|
6086
6185
|
});
|
|
6087
|
-
var replacementEventSchema = z.
|
|
6186
|
+
var replacementEventSchema = z.strictObject({
|
|
6088
6187
|
[IS_MERGE_FIELD2]: customTypes.literalFalse.nullish().describe(isMergeDescription),
|
|
6089
6188
|
[PARENT_ID_FIELD2]: z.string().nullish().describe(
|
|
6090
6189
|
[
|
|
@@ -6100,25 +6199,25 @@ function makeInsertEventSchemas(objectType2, insertSchema) {
|
|
|
6100
6199
|
getEventObjectType(objectType2),
|
|
6101
6200
|
"_"
|
|
6102
6201
|
).replace("_", "");
|
|
6103
|
-
const replaceVariantSchema = insertSchema.merge(replacementEventSchema).
|
|
6104
|
-
const mergeVariantSchema = insertSchema.merge(mergeEventSchema).
|
|
6202
|
+
const replaceVariantSchema = insertSchema.merge(replacementEventSchema).openapi(`Insert${eventSchemaName}EventReplace`);
|
|
6203
|
+
const mergeVariantSchema = insertSchema.merge(mergeEventSchema).openapi(`Insert${eventSchemaName}EventMerge`);
|
|
6105
6204
|
const eventSchema = z.union([replaceVariantSchema, mergeVariantSchema]).describe(`${capitalize(article)} ${eventDescription} event`).openapi(`Insert${eventSchemaName}Event`);
|
|
6106
|
-
const requestSchema = z.
|
|
6205
|
+
const requestSchema = z.strictObject({
|
|
6107
6206
|
events: eventSchema.array().describe(`A list of ${eventDescription} events to insert`)
|
|
6108
|
-
}).
|
|
6207
|
+
}).openapi(`Insert${eventSchemaName}EventRequest`);
|
|
6109
6208
|
return { eventSchema, requestSchema };
|
|
6110
6209
|
}
|
|
6111
|
-
var insertEventsResponseSchema = z.
|
|
6210
|
+
var insertEventsResponseSchema = z.strictObject({
|
|
6112
6211
|
row_ids: z.string().array().describe(
|
|
6113
6212
|
"The ids of all rows that were inserted, aligning one-to-one with the rows provided as input"
|
|
6114
6213
|
)
|
|
6115
|
-
}).
|
|
6214
|
+
}).openapi("InsertEventsResponse");
|
|
6116
6215
|
var {
|
|
6117
6216
|
eventSchema: insertExperimentEventSchema,
|
|
6118
6217
|
requestSchema: insertExperimentEventsRequestSchema
|
|
6119
6218
|
} = makeInsertEventSchemas(
|
|
6120
6219
|
"experiment",
|
|
6121
|
-
z.
|
|
6220
|
+
z.strictObject({
|
|
6122
6221
|
input: experimentEventSchema.shape.input,
|
|
6123
6222
|
output: experimentEventSchema.shape.output,
|
|
6124
6223
|
expected: experimentEventSchema.shape.expected,
|
|
@@ -6131,28 +6230,28 @@ var {
|
|
|
6131
6230
|
id: experimentEventSchema.shape.id.nullish(),
|
|
6132
6231
|
dataset_record_id: experimentEventSchema.shape.dataset_record_id,
|
|
6133
6232
|
[OBJECT_DELETE_FIELD]: experimentEventBaseSchema.shape[OBJECT_DELETE_FIELD]
|
|
6134
|
-
})
|
|
6233
|
+
})
|
|
6135
6234
|
);
|
|
6136
6235
|
var {
|
|
6137
6236
|
eventSchema: insertDatasetEventSchema,
|
|
6138
6237
|
requestSchema: insertDatasetEventsRequestSchema
|
|
6139
6238
|
} = makeInsertEventSchemas(
|
|
6140
6239
|
"dataset",
|
|
6141
|
-
z.
|
|
6240
|
+
z.strictObject({
|
|
6142
6241
|
input: datasetEventSchema.shape.input,
|
|
6143
6242
|
expected: datasetEventSchema.shape.expected,
|
|
6144
6243
|
metadata: datasetEventSchema.shape.metadata,
|
|
6145
6244
|
tags: datasetEventSchema.shape.tags,
|
|
6146
6245
|
id: datasetEventSchema.shape.id.nullish(),
|
|
6147
6246
|
[OBJECT_DELETE_FIELD]: datasetEventBaseSchema.shape[OBJECT_DELETE_FIELD]
|
|
6148
|
-
})
|
|
6247
|
+
})
|
|
6149
6248
|
);
|
|
6150
6249
|
var {
|
|
6151
6250
|
eventSchema: insertProjectLogsEventSchema,
|
|
6152
6251
|
requestSchema: insertProjectLogsEventsRequestSchema
|
|
6153
6252
|
} = makeInsertEventSchemas(
|
|
6154
6253
|
"project",
|
|
6155
|
-
z.
|
|
6254
|
+
z.strictObject({
|
|
6156
6255
|
input: projectLogsEventSchema.shape.input,
|
|
6157
6256
|
output: projectLogsEventSchema.shape.output,
|
|
6158
6257
|
expected: projectLogsEventSchema.shape.expected,
|
|
@@ -6164,7 +6263,7 @@ var {
|
|
|
6164
6263
|
span_attributes: projectLogsEventSchema.shape.span_attributes,
|
|
6165
6264
|
id: projectLogsEventSchema.shape.id.nullish(),
|
|
6166
6265
|
[OBJECT_DELETE_FIELD]: projectLogsEventBaseSchema.shape[OBJECT_DELETE_FIELD]
|
|
6167
|
-
})
|
|
6266
|
+
})
|
|
6168
6267
|
);
|
|
6169
6268
|
function makeFeedbackRequestSchema(objectType2, feedbackSchema) {
|
|
6170
6269
|
const eventDescription = getEventObjectDescription(objectType2);
|
|
@@ -6172,54 +6271,54 @@ function makeFeedbackRequestSchema(objectType2, feedbackSchema) {
|
|
|
6172
6271
|
getEventObjectType(objectType2),
|
|
6173
6272
|
"_"
|
|
6174
6273
|
).replace("_", "");
|
|
6175
|
-
return z.
|
|
6274
|
+
return z.strictObject({
|
|
6176
6275
|
feedback: feedbackSchema.array().describe(`A list of ${eventDescription} feedback items`)
|
|
6177
|
-
}).
|
|
6276
|
+
}).openapi(`Feedback${eventSchemaName}EventRequest`);
|
|
6178
6277
|
}
|
|
6179
6278
|
var feedbackExperimentRequestBaseSchema = generateBaseEventFeedbackSchema("experiment");
|
|
6180
|
-
var feedbackExperimentItemSchema = z.
|
|
6279
|
+
var feedbackExperimentItemSchema = z.strictObject({
|
|
6181
6280
|
id: feedbackExperimentRequestBaseSchema.shape.id,
|
|
6182
6281
|
scores: feedbackExperimentRequestBaseSchema.shape.scores,
|
|
6183
6282
|
expected: feedbackExperimentRequestBaseSchema.shape.expected,
|
|
6184
6283
|
comment: feedbackExperimentRequestBaseSchema.shape.comment,
|
|
6185
6284
|
metadata: feedbackExperimentRequestBaseSchema.shape.metadata,
|
|
6186
6285
|
source: feedbackExperimentRequestBaseSchema.shape.source
|
|
6187
|
-
}).
|
|
6286
|
+
}).openapi("FeedbackExperimentItem");
|
|
6188
6287
|
var feedbackExperimentRequestSchema = makeFeedbackRequestSchema(
|
|
6189
6288
|
"experiment",
|
|
6190
6289
|
feedbackExperimentItemSchema
|
|
6191
6290
|
);
|
|
6192
6291
|
var feedbackDatasetRequestBaseSchema = generateBaseEventFeedbackSchema("dataset");
|
|
6193
|
-
var feedbackDatasetItemSchema = z.
|
|
6292
|
+
var feedbackDatasetItemSchema = z.strictObject({
|
|
6194
6293
|
id: feedbackDatasetRequestBaseSchema.shape.id,
|
|
6195
6294
|
comment: feedbackDatasetRequestBaseSchema.shape.comment,
|
|
6196
6295
|
metadata: feedbackDatasetRequestBaseSchema.shape.metadata,
|
|
6197
6296
|
source: feedbackDatasetRequestBaseSchema.shape.source
|
|
6198
|
-
}).
|
|
6297
|
+
}).openapi("FeedbackDatasetItem");
|
|
6199
6298
|
var feedbackDatasetRequestSchema = makeFeedbackRequestSchema(
|
|
6200
6299
|
"dataset",
|
|
6201
6300
|
feedbackDatasetItemSchema
|
|
6202
6301
|
);
|
|
6203
6302
|
var feedbackProjectLogsRequestBaseSchema = generateBaseEventFeedbackSchema("project");
|
|
6204
|
-
var feedbackProjectLogsItemSchema = z.
|
|
6303
|
+
var feedbackProjectLogsItemSchema = z.strictObject({
|
|
6205
6304
|
id: feedbackProjectLogsRequestBaseSchema.shape.id,
|
|
6206
6305
|
scores: feedbackProjectLogsRequestBaseSchema.shape.scores,
|
|
6207
6306
|
expected: feedbackProjectLogsRequestBaseSchema.shape.expected,
|
|
6208
6307
|
comment: feedbackProjectLogsRequestBaseSchema.shape.comment,
|
|
6209
6308
|
metadata: feedbackProjectLogsRequestBaseSchema.shape.metadata,
|
|
6210
6309
|
source: feedbackProjectLogsRequestBaseSchema.shape.source
|
|
6211
|
-
}).
|
|
6310
|
+
}).openapi("FeedbackProjectLogsItem");
|
|
6212
6311
|
var feedbackProjectLogsRequestSchema = makeFeedbackRequestSchema(
|
|
6213
6312
|
"project",
|
|
6214
6313
|
feedbackProjectLogsItemSchema
|
|
6215
6314
|
);
|
|
6216
6315
|
var feedbackPromptRequestBaseSchema = generateBaseEventFeedbackSchema("prompt");
|
|
6217
|
-
var feedbackPromptItemSchema = z.
|
|
6316
|
+
var feedbackPromptItemSchema = z.strictObject({
|
|
6218
6317
|
id: feedbackPromptRequestBaseSchema.shape.id,
|
|
6219
6318
|
comment: feedbackPromptRequestBaseSchema.shape.comment,
|
|
6220
6319
|
metadata: feedbackPromptRequestBaseSchema.shape.metadata,
|
|
6221
6320
|
source: feedbackPromptRequestBaseSchema.shape.source
|
|
6222
|
-
}).
|
|
6321
|
+
}).openapi("FeedbackPromptItem");
|
|
6223
6322
|
var feedbackPromptRequestSchema = makeFeedbackRequestSchema(
|
|
6224
6323
|
"prompt",
|
|
6225
6324
|
feedbackPromptItemSchema
|
|
@@ -6264,12 +6363,12 @@ function makeCrossObjectIndividualRequestSchema(objectType2) {
|
|
|
6264
6363
|
const eventObjectType = getEventObjectType(objectType2);
|
|
6265
6364
|
const eventDescription = getEventObjectDescription(objectType2);
|
|
6266
6365
|
const eventObjectSchema = eventObjectSchemas[eventObjectType];
|
|
6267
|
-
const insertObject = z.
|
|
6366
|
+
const insertObject = z.strictObject({
|
|
6268
6367
|
...eventObjectSchema.insertEvent ? {
|
|
6269
6368
|
events: eventObjectSchema.insertEvent.array().nullish().describe(`A list of ${eventDescription} events to insert`)
|
|
6270
6369
|
} : {},
|
|
6271
6370
|
feedback: eventObjectSchema.feedbackItem.array().nullish().describe(`A list of ${eventDescription} feedback items`)
|
|
6272
|
-
})
|
|
6371
|
+
});
|
|
6273
6372
|
return z.record(z.string().uuid(), insertObject).nullish().describe(
|
|
6274
6373
|
`A mapping from ${objectType2} id to a set of log events and feedback items to insert`
|
|
6275
6374
|
);
|
|
@@ -6279,18 +6378,16 @@ function makeCrossObjectIndividualResponseSchema(objectType2) {
|
|
|
6279
6378
|
`A mapping from ${objectType2} id to row ids for inserted \`events\``
|
|
6280
6379
|
);
|
|
6281
6380
|
}
|
|
6282
|
-
var crossObjectInsertRequestSchema = z.
|
|
6381
|
+
var crossObjectInsertRequestSchema = z.strictObject({
|
|
6283
6382
|
experiment: makeCrossObjectIndividualRequestSchema("experiment"),
|
|
6284
6383
|
dataset: makeCrossObjectIndividualRequestSchema("dataset"),
|
|
6285
|
-
project_logs: makeCrossObjectIndividualRequestSchema("project")
|
|
6286
|
-
|
|
6287
|
-
|
|
6288
|
-
var crossObjectInsertResponseSchema = z.object({
|
|
6384
|
+
project_logs: makeCrossObjectIndividualRequestSchema("project")
|
|
6385
|
+
}).openapi("CrossObjectInsertRequest");
|
|
6386
|
+
var crossObjectInsertResponseSchema = z.strictObject({
|
|
6289
6387
|
experiment: makeCrossObjectIndividualResponseSchema("experiment"),
|
|
6290
6388
|
dataset: makeCrossObjectIndividualResponseSchema("dataset"),
|
|
6291
|
-
project_logs: makeCrossObjectIndividualResponseSchema("project")
|
|
6292
|
-
|
|
6293
|
-
}).strict().openapi("CrossObjectInsertResponse");
|
|
6389
|
+
project_logs: makeCrossObjectIndividualResponseSchema("project")
|
|
6390
|
+
}).openapi("CrossObjectInsertResponse");
|
|
6294
6391
|
var summarizeScoresParamSchema = z.boolean().describe(
|
|
6295
6392
|
"Whether to summarize the scores and metrics. If false (or omitted), only the metadata will be returned."
|
|
6296
6393
|
);
|
|
@@ -6300,14 +6397,14 @@ var comparisonExperimentIdParamSchema = z.string().uuid().describe(
|
|
|
6300
6397
|
var summarizeDataParamSchema = z.boolean().describe(
|
|
6301
6398
|
"Whether to summarize the data. If false (or omitted), only the metadata will be returned."
|
|
6302
6399
|
);
|
|
6303
|
-
var summarizeExperimentResponseSchema = z.
|
|
6400
|
+
var summarizeExperimentResponseSchema = z.strictObject({
|
|
6304
6401
|
project_name: z.string().describe("Name of the project that the experiment belongs to"),
|
|
6305
6402
|
experiment_name: z.string().describe("Name of the experiment"),
|
|
6306
6403
|
project_url: z.string().url().describe("URL to the project's page in the Braintrust app"),
|
|
6307
6404
|
experiment_url: z.string().url().describe("URL to the experiment's page in the Braintrust app"),
|
|
6308
6405
|
comparison_experiment_name: z.string().nullish().describe("The experiment which scores are baselined against"),
|
|
6309
6406
|
scores: z.record(
|
|
6310
|
-
z.
|
|
6407
|
+
z.strictObject({
|
|
6311
6408
|
name: z.string().describe("Name of the score"),
|
|
6312
6409
|
score: z.number().min(0).max(1).describe("Average score across all examples"),
|
|
6313
6410
|
diff: z.number().min(-1).max(1).describe(
|
|
@@ -6318,7 +6415,7 @@ var summarizeExperimentResponseSchema = z.object({
|
|
|
6318
6415
|
}).describe("Summary of a score's performance").openapi("ScoreSummary")
|
|
6319
6416
|
).nullish().describe("Summary of the experiment's scores"),
|
|
6320
6417
|
metrics: z.record(
|
|
6321
|
-
z.
|
|
6418
|
+
z.strictObject({
|
|
6322
6419
|
name: z.string().describe("Name of the metric"),
|
|
6323
6420
|
metric: z.number().describe("Average metric across all examples"),
|
|
6324
6421
|
unit: z.string().describe("Unit label for the metric"),
|
|
@@ -6329,16 +6426,16 @@ var summarizeExperimentResponseSchema = z.object({
|
|
|
6329
6426
|
regressions: z.number().int().min(0).describe("Number of regressions in the metric")
|
|
6330
6427
|
}).describe("Summary of a metric's performance").openapi("MetricSummary")
|
|
6331
6428
|
).nullish().describe("Summary of the experiment's metrics")
|
|
6332
|
-
}).
|
|
6333
|
-
var summarizeDatasetResponseSchema = z.
|
|
6429
|
+
}).describe("Summary of an experiment").openapi("SummarizeExperimentResponse");
|
|
6430
|
+
var summarizeDatasetResponseSchema = z.strictObject({
|
|
6334
6431
|
project_name: z.string().describe("Name of the project that the dataset belongs to"),
|
|
6335
6432
|
dataset_name: z.string().describe("Name of the dataset"),
|
|
6336
6433
|
project_url: z.string().url().describe("URL to the project's page in the Braintrust app"),
|
|
6337
6434
|
dataset_url: z.string().url().describe("URL to the dataset's page in the Braintrust app"),
|
|
6338
|
-
data_summary: z.
|
|
6435
|
+
data_summary: z.strictObject({
|
|
6339
6436
|
total_records: z.number().int().min(0).describe("Total number of records in the dataset")
|
|
6340
6437
|
}).nullish().describe("Summary of a dataset's data").openapi("DataSummary")
|
|
6341
|
-
}).
|
|
6438
|
+
}).describe("Summary of a dataset").openapi("SummarizeDatasetResponse");
|
|
6342
6439
|
|
|
6343
6440
|
// src/util.ts
|
|
6344
6441
|
var GLOBAL_PROJECT = "Global";
|
|
@@ -6852,7 +6949,7 @@ var NoopSpan = class {
|
|
|
6852
6949
|
}
|
|
6853
6950
|
log(_) {
|
|
6854
6951
|
}
|
|
6855
|
-
logFeedback(
|
|
6952
|
+
logFeedback(_event) {
|
|
6856
6953
|
}
|
|
6857
6954
|
traced(callback, _1) {
|
|
6858
6955
|
return callback(this);
|
|
@@ -6866,6 +6963,8 @@ var NoopSpan = class {
|
|
|
6866
6963
|
close(args) {
|
|
6867
6964
|
return this.end(args);
|
|
6868
6965
|
}
|
|
6966
|
+
async flush() {
|
|
6967
|
+
}
|
|
6869
6968
|
};
|
|
6870
6969
|
var NOOP_SPAN = new NoopSpan();
|
|
6871
6970
|
var BraintrustState = class {
|
|
@@ -6875,7 +6974,12 @@ var BraintrustState = class {
|
|
|
6875
6974
|
// (safely) dynamically cast it whenever retrieving the logger.
|
|
6876
6975
|
currentLogger;
|
|
6877
6976
|
currentSpan;
|
|
6977
|
+
// For operations not tied to a particular data object instance, we maintain a
|
|
6978
|
+
// global BackgroundLogger which is refreshed anytime we log in with different
|
|
6979
|
+
// credentials.
|
|
6980
|
+
_globalBgLogger = void 0;
|
|
6878
6981
|
appUrl = null;
|
|
6982
|
+
appPublicUrl = null;
|
|
6879
6983
|
loginToken = null;
|
|
6880
6984
|
orgId = null;
|
|
6881
6985
|
orgName = null;
|
|
@@ -6884,6 +6988,7 @@ var BraintrustState = class {
|
|
|
6884
6988
|
gitMetadataSettings;
|
|
6885
6989
|
_apiConn = null;
|
|
6886
6990
|
_logConn = null;
|
|
6991
|
+
_globalBgLoggerDirty = true;
|
|
6887
6992
|
constructor() {
|
|
6888
6993
|
this.id = v4_default();
|
|
6889
6994
|
this.currentExperiment = void 0;
|
|
@@ -6902,6 +7007,7 @@ var BraintrustState = class {
|
|
|
6902
7007
|
this.gitMetadataSettings = void 0;
|
|
6903
7008
|
this._apiConn = null;
|
|
6904
7009
|
this._logConn = null;
|
|
7010
|
+
this._globalBgLoggerDirty = true;
|
|
6905
7011
|
}
|
|
6906
7012
|
apiConn() {
|
|
6907
7013
|
if (!this._apiConn) {
|
|
@@ -6921,6 +7027,18 @@ var BraintrustState = class {
|
|
|
6921
7027
|
}
|
|
6922
7028
|
return this._logConn;
|
|
6923
7029
|
}
|
|
7030
|
+
globalBgLogger() {
|
|
7031
|
+
if (this._globalBgLogger === void 0 || this._globalBgLoggerDirty) {
|
|
7032
|
+
const getLogConn = async () => {
|
|
7033
|
+
await login();
|
|
7034
|
+
this._globalBgLoggerDirty = false;
|
|
7035
|
+
return this.logConn();
|
|
7036
|
+
};
|
|
7037
|
+
this._globalBgLogger = new BackgroundLogger(new LazyValue(getLogConn));
|
|
7038
|
+
this._globalBgLoggerDirty = false;
|
|
7039
|
+
}
|
|
7040
|
+
return this._globalBgLogger;
|
|
7041
|
+
}
|
|
6924
7042
|
};
|
|
6925
7043
|
var _state;
|
|
6926
7044
|
function _internalSetInitialState() {
|
|
@@ -7043,7 +7161,7 @@ var HTTPConnection = class _HTTPConnection {
|
|
|
7043
7161
|
return await resp.json();
|
|
7044
7162
|
}
|
|
7045
7163
|
};
|
|
7046
|
-
function logFeedbackImpl(bgLogger,
|
|
7164
|
+
function logFeedbackImpl(bgLogger, parentObjectType, parentObjectId, {
|
|
7047
7165
|
id,
|
|
7048
7166
|
expected,
|
|
7049
7167
|
scores,
|
|
@@ -7071,16 +7189,17 @@ function logFeedbackImpl(bgLogger, parentIds, {
|
|
|
7071
7189
|
updateEvent = Object.fromEntries(
|
|
7072
7190
|
Object.entries(updateEvent).filter(([_, v]) => !isEmpty(v))
|
|
7073
7191
|
);
|
|
7074
|
-
const
|
|
7075
|
-
|
|
7076
|
-
|
|
7077
|
-
|
|
7192
|
+
const parentIds = async () => new SpanParentComponents({
|
|
7193
|
+
objectType: parentObjectType,
|
|
7194
|
+
objectId: await parentObjectId.get(),
|
|
7195
|
+
rowId: ""
|
|
7196
|
+
}).asDict();
|
|
7078
7197
|
if (Object.keys(updateEvent).length > 0) {
|
|
7079
7198
|
const record = new LazyValue(async () => {
|
|
7080
7199
|
return {
|
|
7081
7200
|
id,
|
|
7082
7201
|
...updateEvent,
|
|
7083
|
-
...await
|
|
7202
|
+
...await parentIds(),
|
|
7084
7203
|
[AUDIT_SOURCE_FIELD]: source,
|
|
7085
7204
|
[AUDIT_METADATA_FIELD]: metadata,
|
|
7086
7205
|
[IS_MERGE_FIELD]: true
|
|
@@ -7101,7 +7220,7 @@ function logFeedbackImpl(bgLogger, parentIds, {
|
|
|
7101
7220
|
comment: {
|
|
7102
7221
|
text: comment
|
|
7103
7222
|
},
|
|
7104
|
-
...await
|
|
7223
|
+
...await parentIds(),
|
|
7105
7224
|
[AUDIT_SOURCE_FIELD]: source,
|
|
7106
7225
|
[AUDIT_METADATA_FIELD]: metadata
|
|
7107
7226
|
};
|
|
@@ -7109,11 +7228,48 @@ function logFeedbackImpl(bgLogger, parentIds, {
|
|
|
7109
7228
|
bgLogger.log([record]);
|
|
7110
7229
|
}
|
|
7111
7230
|
}
|
|
7231
|
+
function startSpanParentArgs(args) {
|
|
7232
|
+
if (args.parent && args.parentId) {
|
|
7233
|
+
throw new Error(
|
|
7234
|
+
"Cannot specify both `parent` and `parentId`. Prefer `parent"
|
|
7235
|
+
);
|
|
7236
|
+
}
|
|
7237
|
+
let parentObjectId = void 0;
|
|
7238
|
+
let parentRowId = void 0;
|
|
7239
|
+
if (args.parent) {
|
|
7240
|
+
const parentComponents = SpanParentComponents.fromStr(args.parent);
|
|
7241
|
+
if (args.spanParentObjectType !== parentComponents.objectType) {
|
|
7242
|
+
throw new Error(
|
|
7243
|
+
`Mismatch between expected span parent object type ${args.spanParentObjectType} and provided type ${parentComponents.objectType}`
|
|
7244
|
+
);
|
|
7245
|
+
}
|
|
7246
|
+
const computeParentObjectId = async () => {
|
|
7247
|
+
if (await args.spanParentObjectId.get() !== parentComponents.objectId) {
|
|
7248
|
+
throw new Error(
|
|
7249
|
+
`Mismatch between expected span parent object id ${await args.spanParentObjectId.get()} and provided id ${parentComponents.objectId}`
|
|
7250
|
+
);
|
|
7251
|
+
}
|
|
7252
|
+
return await args.spanParentObjectId.get();
|
|
7253
|
+
};
|
|
7254
|
+
parentObjectId = new LazyValue(computeParentObjectId);
|
|
7255
|
+
parentRowId = parentComponents.rowId;
|
|
7256
|
+
} else {
|
|
7257
|
+
parentObjectId = args.spanParentObjectId;
|
|
7258
|
+
parentRowId = args.parentId ?? "";
|
|
7259
|
+
}
|
|
7260
|
+
return {
|
|
7261
|
+
parentObjectType: args.spanParentObjectType,
|
|
7262
|
+
parentObjectId,
|
|
7263
|
+
parentRowId
|
|
7264
|
+
};
|
|
7265
|
+
}
|
|
7112
7266
|
var Logger = class {
|
|
7113
7267
|
lazyMetadata;
|
|
7114
7268
|
logOptions;
|
|
7115
7269
|
bgLogger;
|
|
7116
7270
|
lastStartTime;
|
|
7271
|
+
lazyId;
|
|
7272
|
+
calledStartSpan;
|
|
7117
7273
|
// For type identification.
|
|
7118
7274
|
kind = "logger";
|
|
7119
7275
|
constructor(lazyMetadata, logOptions = {}) {
|
|
@@ -7124,6 +7280,8 @@ var Logger = class {
|
|
|
7124
7280
|
);
|
|
7125
7281
|
this.bgLogger = new BackgroundLogger(logConn);
|
|
7126
7282
|
this.lastStartTime = getCurrentUnixTimestamp();
|
|
7283
|
+
this.lazyId = new LazyValue(async () => await this.id);
|
|
7284
|
+
this.calledStartSpan = false;
|
|
7127
7285
|
}
|
|
7128
7286
|
get org_id() {
|
|
7129
7287
|
return (async () => {
|
|
@@ -7135,6 +7293,12 @@ var Logger = class {
|
|
|
7135
7293
|
return (await this.lazyMetadata.get()).project;
|
|
7136
7294
|
})();
|
|
7137
7295
|
}
|
|
7296
|
+
get id() {
|
|
7297
|
+
return (async () => (await this.project).id)();
|
|
7298
|
+
}
|
|
7299
|
+
spanParentObjectType() {
|
|
7300
|
+
return SpanParentObjectType.PROJECT_LOGS;
|
|
7301
|
+
}
|
|
7138
7302
|
async getState() {
|
|
7139
7303
|
await this.lazyMetadata.get();
|
|
7140
7304
|
return _state;
|
|
@@ -7151,19 +7315,16 @@ var Logger = class {
|
|
|
7151
7315
|
* @param event.metrics: (Optional) a dictionary of metrics to log. The following keys are populated automatically: "start", "end".
|
|
7152
7316
|
* @param event.id: (Optional) a unique identifier for the event. If you don't provide one, BrainTrust will generate one for you.
|
|
7153
7317
|
* @param options Additional logging options
|
|
7154
|
-
* @param options.
|
|
7318
|
+
* @param options.allowConcurrentWithSpans in rare cases where you need to log at the top level separately from spans on the logger elsewhere, set this to true.
|
|
7155
7319
|
* :returns: The `id` of the logged event.
|
|
7156
7320
|
*/
|
|
7157
7321
|
log(event, options) {
|
|
7158
|
-
if (!options?.
|
|
7159
|
-
|
|
7160
|
-
|
|
7161
|
-
|
|
7162
|
-
"Cannot run toplevel Logger.log method while there is an active span. To log to the span, use Span.log"
|
|
7163
|
-
);
|
|
7164
|
-
}
|
|
7322
|
+
if (this.calledStartSpan && !options?.allowConcurrentWithSpans) {
|
|
7323
|
+
throw new Error(
|
|
7324
|
+
"Cannot run toplevel `log` method while using spans. To log to the span, call `logger.traced` and then log with `span.log`"
|
|
7325
|
+
);
|
|
7165
7326
|
}
|
|
7166
|
-
const span = this.
|
|
7327
|
+
const span = this.startSpanImpl({ startTime: this.lastStartTime, event });
|
|
7167
7328
|
this.lastStartTime = span.end();
|
|
7168
7329
|
const ret = span.id;
|
|
7169
7330
|
if (this.logOptions.asyncFlush === true) {
|
|
@@ -7203,14 +7364,6 @@ var Logger = class {
|
|
|
7203
7364
|
})();
|
|
7204
7365
|
}
|
|
7205
7366
|
}
|
|
7206
|
-
async lazyParentIds() {
|
|
7207
|
-
return {
|
|
7208
|
-
kind: "project_log",
|
|
7209
|
-
org_id: await this.org_id,
|
|
7210
|
-
project_id: (await this.project).id,
|
|
7211
|
-
log_id: "g"
|
|
7212
|
-
};
|
|
7213
|
-
}
|
|
7214
7367
|
/**
|
|
7215
7368
|
* Lower-level alternative to `traced`. This allows you to start a span yourself, and can be useful in situations
|
|
7216
7369
|
* where you cannot use callbacks. However, spans started with `startSpan` will not be marked as the "current span",
|
|
@@ -7219,13 +7372,20 @@ var Logger = class {
|
|
|
7219
7372
|
* See `traced` for full details.
|
|
7220
7373
|
*/
|
|
7221
7374
|
startSpan(args) {
|
|
7222
|
-
|
|
7375
|
+
this.calledStartSpan = true;
|
|
7376
|
+
return this.startSpanImpl(args);
|
|
7377
|
+
}
|
|
7378
|
+
startSpanImpl(args) {
|
|
7223
7379
|
return new SpanImpl({
|
|
7224
|
-
|
|
7225
|
-
|
|
7380
|
+
...startSpanParentArgs({
|
|
7381
|
+
parent: args?.parent,
|
|
7382
|
+
parentId: args?.parentId,
|
|
7383
|
+
spanParentObjectType: this.spanParentObjectType(),
|
|
7384
|
+
spanParentObjectId: this.lazyId
|
|
7385
|
+
}),
|
|
7386
|
+
...args,
|
|
7226
7387
|
bgLogger: this.bgLogger,
|
|
7227
|
-
|
|
7228
|
-
...argsRest
|
|
7388
|
+
defaultRootType: SpanTypeAttribute.TASK
|
|
7229
7389
|
});
|
|
7230
7390
|
}
|
|
7231
7391
|
/**
|
|
@@ -7242,10 +7402,21 @@ var Logger = class {
|
|
|
7242
7402
|
logFeedback(event) {
|
|
7243
7403
|
logFeedbackImpl(
|
|
7244
7404
|
this.bgLogger,
|
|
7245
|
-
|
|
7405
|
+
this.spanParentObjectType(),
|
|
7406
|
+
this.lazyId,
|
|
7246
7407
|
event
|
|
7247
7408
|
);
|
|
7248
7409
|
}
|
|
7410
|
+
/**
|
|
7411
|
+
* Return a serialized representation of the logger that can be used to start subspans in other places. See `Span.start_span` for more details.
|
|
7412
|
+
*/
|
|
7413
|
+
async export() {
|
|
7414
|
+
return new SpanParentComponents({
|
|
7415
|
+
objectType: this.spanParentObjectType(),
|
|
7416
|
+
objectId: await this.id,
|
|
7417
|
+
rowId: ""
|
|
7418
|
+
}).toStr();
|
|
7419
|
+
}
|
|
7249
7420
|
/*
|
|
7250
7421
|
* Flush any pending logs to the server.
|
|
7251
7422
|
*/
|
|
@@ -7835,8 +8006,10 @@ async function login(options = {}) {
|
|
|
7835
8006
|
apiKey = isomorph_default.getEnv("BRAINTRUST_API_KEY"),
|
|
7836
8007
|
orgName = isomorph_default.getEnv("BRAINTRUST_ORG_NAME")
|
|
7837
8008
|
} = options || {};
|
|
8009
|
+
const appPublicUrl = isomorph_default.getEnv("BRAINTRUST_APP_PUBLIC_URL") || appUrl;
|
|
7838
8010
|
_state.resetLoginInfo();
|
|
7839
8011
|
_state.appUrl = appUrl;
|
|
8012
|
+
_state.appPublicUrl = appPublicUrl;
|
|
7840
8013
|
let conn = null;
|
|
7841
8014
|
if (apiKey !== void 0) {
|
|
7842
8015
|
const resp = await checkResponse(
|
|
@@ -7912,7 +8085,33 @@ function getSpanParentObject(options) {
|
|
|
7912
8085
|
return NOOP_SPAN;
|
|
7913
8086
|
}
|
|
7914
8087
|
function traced(callback, args) {
|
|
7915
|
-
const { span,
|
|
8088
|
+
const { span, isLogger } = (() => {
|
|
8089
|
+
if (args?.parent) {
|
|
8090
|
+
if (args?.parentId) {
|
|
8091
|
+
throw new Error(
|
|
8092
|
+
"Cannot specify both `parent` and `parent_id`. Prefer `parent`"
|
|
8093
|
+
);
|
|
8094
|
+
}
|
|
8095
|
+
const components = SpanParentComponents.fromStr(args?.parent);
|
|
8096
|
+
const span2 = new SpanImpl({
|
|
8097
|
+
...args,
|
|
8098
|
+
parentObjectType: components.objectType,
|
|
8099
|
+
parentObjectId: new LazyValue(async () => components.objectId),
|
|
8100
|
+
parentRowId: components.rowId,
|
|
8101
|
+
bgLogger: _state.globalBgLogger()
|
|
8102
|
+
});
|
|
8103
|
+
return {
|
|
8104
|
+
span: span2,
|
|
8105
|
+
isLogger: components.objectType === SpanParentObjectType.PROJECT_LOGS
|
|
8106
|
+
};
|
|
8107
|
+
} else {
|
|
8108
|
+
const parentObject = getSpanParentObject({
|
|
8109
|
+
asyncFlush: args?.asyncFlush
|
|
8110
|
+
});
|
|
8111
|
+
const span2 = parentObject.startSpan(args);
|
|
8112
|
+
return { span: span2, isLogger: parentObject.kind === "logger" };
|
|
8113
|
+
}
|
|
8114
|
+
})();
|
|
7916
8115
|
const ret = runFinally(
|
|
7917
8116
|
() => {
|
|
7918
8117
|
if (args?.setCurrent ?? true) {
|
|
@@ -7928,23 +8127,17 @@ function traced(callback, args) {
|
|
|
7928
8127
|
} else {
|
|
7929
8128
|
return (async () => {
|
|
7930
8129
|
const awaitedRet = await ret;
|
|
7931
|
-
if (
|
|
7932
|
-
await
|
|
8130
|
+
if (isLogger) {
|
|
8131
|
+
await span.flush();
|
|
7933
8132
|
}
|
|
7934
8133
|
return awaitedRet;
|
|
7935
8134
|
})();
|
|
7936
8135
|
}
|
|
7937
8136
|
}
|
|
7938
8137
|
function startSpan(args) {
|
|
7939
|
-
return
|
|
7940
|
-
}
|
|
7941
|
-
function startSpanReturnParent(args) {
|
|
7942
|
-
const parentObject = getSpanParentObject({
|
|
8138
|
+
return getSpanParentObject({
|
|
7943
8139
|
asyncFlush: args?.asyncFlush
|
|
7944
|
-
});
|
|
7945
|
-
const { name: nameOpt, ...argsRest } = args ?? {};
|
|
7946
|
-
const name = parentObject.kind === "span" ? nameOpt : nameOpt ?? "root";
|
|
7947
|
-
return { span: parentObject.startSpan({ name, ...argsRest }), parentObject };
|
|
8140
|
+
}).startSpan(args);
|
|
7948
8141
|
}
|
|
7949
8142
|
function withCurrent(span, callback) {
|
|
7950
8143
|
return _state.currentSpan.run(span, () => callback(span));
|
|
@@ -8126,6 +8319,8 @@ var Experiment = class extends ObjectFetcher {
|
|
|
8126
8319
|
dataset;
|
|
8127
8320
|
bgLogger;
|
|
8128
8321
|
lastStartTime;
|
|
8322
|
+
lazyId;
|
|
8323
|
+
calledStartSpan;
|
|
8129
8324
|
// For type identification.
|
|
8130
8325
|
kind = "experiment";
|
|
8131
8326
|
constructor(lazyMetadata, dataset) {
|
|
@@ -8137,6 +8332,8 @@ var Experiment = class extends ObjectFetcher {
|
|
|
8137
8332
|
);
|
|
8138
8333
|
this.bgLogger = new BackgroundLogger(logConn);
|
|
8139
8334
|
this.lastStartTime = getCurrentUnixTimestamp();
|
|
8335
|
+
this.lazyId = new LazyValue(async () => await this.id);
|
|
8336
|
+
this.calledStartSpan = false;
|
|
8140
8337
|
}
|
|
8141
8338
|
get id() {
|
|
8142
8339
|
return (async () => {
|
|
@@ -8153,6 +8350,9 @@ var Experiment = class extends ObjectFetcher {
|
|
|
8153
8350
|
return (await this.lazyMetadata.get()).project;
|
|
8154
8351
|
})();
|
|
8155
8352
|
}
|
|
8353
|
+
spanParentObjectType() {
|
|
8354
|
+
return SpanParentObjectType.EXPERIMENT;
|
|
8355
|
+
}
|
|
8156
8356
|
async getState() {
|
|
8157
8357
|
await this.lazyMetadata.get();
|
|
8158
8358
|
return _state;
|
|
@@ -8171,20 +8371,17 @@ var Experiment = class extends ObjectFetcher {
|
|
|
8171
8371
|
* @param event.dataset_record_id: (Optional) the id of the dataset record that this event is associated with. This field is required if and only if the experiment is associated with a dataset.
|
|
8172
8372
|
* @param event.inputs: (Deprecated) the same as `input` (will be removed in a future version).
|
|
8173
8373
|
* @param options Additional logging options
|
|
8174
|
-
* @param options.
|
|
8374
|
+
* @param options.allowConcurrentWithSpans in rare cases where you need to log at the top level separately from spans on the experiment elsewhere, set this to true.
|
|
8175
8375
|
* :returns: The `id` of the logged event.
|
|
8176
8376
|
*/
|
|
8177
8377
|
log(event, options) {
|
|
8178
|
-
if (!options?.
|
|
8179
|
-
|
|
8180
|
-
|
|
8181
|
-
|
|
8182
|
-
"Cannot run toplevel Experiment.log method while there is an active span. To log to the span, use Span.log"
|
|
8183
|
-
);
|
|
8184
|
-
}
|
|
8378
|
+
if (this.calledStartSpan && !options?.allowConcurrentWithSpans) {
|
|
8379
|
+
throw new Error(
|
|
8380
|
+
"Cannot run toplevel `log` method while using spans. To log to the span, call `experiment.traced` and then log with `span.log`"
|
|
8381
|
+
);
|
|
8185
8382
|
}
|
|
8186
8383
|
event = validateAndSanitizeExperimentLogFullArgs(event, !!this.dataset);
|
|
8187
|
-
const span = this.
|
|
8384
|
+
const span = this.startSpanImpl({ startTime: this.lastStartTime, event });
|
|
8188
8385
|
this.lastStartTime = span.end();
|
|
8189
8386
|
return span.id;
|
|
8190
8387
|
}
|
|
@@ -8207,13 +8404,6 @@ var Experiment = class extends ObjectFetcher {
|
|
|
8207
8404
|
() => span.end()
|
|
8208
8405
|
);
|
|
8209
8406
|
}
|
|
8210
|
-
async lazyParentIds() {
|
|
8211
|
-
return {
|
|
8212
|
-
kind: "experiment",
|
|
8213
|
-
project_id: (await this.project).id,
|
|
8214
|
-
experiment_id: await this.id
|
|
8215
|
-
};
|
|
8216
|
-
}
|
|
8217
8407
|
/**
|
|
8218
8408
|
* Lower-level alternative to `traced`. This allows you to start a span yourself, and can be useful in situations
|
|
8219
8409
|
* where you cannot use callbacks. However, spans started with `startSpan` will not be marked as the "current span",
|
|
@@ -8222,13 +8412,20 @@ var Experiment = class extends ObjectFetcher {
|
|
|
8222
8412
|
* See `traced` for full details.
|
|
8223
8413
|
*/
|
|
8224
8414
|
startSpan(args) {
|
|
8225
|
-
|
|
8415
|
+
this.calledStartSpan = true;
|
|
8416
|
+
return this.startSpanImpl(args);
|
|
8417
|
+
}
|
|
8418
|
+
startSpanImpl(args) {
|
|
8226
8419
|
return new SpanImpl({
|
|
8227
|
-
|
|
8228
|
-
|
|
8420
|
+
...startSpanParentArgs({
|
|
8421
|
+
parent: args?.parent,
|
|
8422
|
+
parentId: args?.parentId,
|
|
8423
|
+
spanParentObjectType: this.spanParentObjectType(),
|
|
8424
|
+
spanParentObjectId: this.lazyId
|
|
8425
|
+
}),
|
|
8426
|
+
...args,
|
|
8229
8427
|
bgLogger: this.bgLogger,
|
|
8230
|
-
|
|
8231
|
-
...argsRest
|
|
8428
|
+
defaultRootType: SpanTypeAttribute.EVAL
|
|
8232
8429
|
});
|
|
8233
8430
|
}
|
|
8234
8431
|
async fetchBaseExperiment() {
|
|
@@ -8263,10 +8460,10 @@ var Experiment = class extends ObjectFetcher {
|
|
|
8263
8460
|
let { summarizeScores = true, comparisonExperimentId = void 0 } = options || {};
|
|
8264
8461
|
await this.bgLogger.flush();
|
|
8265
8462
|
const state = await this.getState();
|
|
8266
|
-
const projectUrl = `${state.
|
|
8463
|
+
const projectUrl = `${state.appPublicUrl}/app/${encodeURIComponent(
|
|
8267
8464
|
state.orgName
|
|
8268
8465
|
)}/p/${encodeURIComponent((await this.project).name)}`;
|
|
8269
|
-
const experimentUrl = `${projectUrl}/${encodeURIComponent(
|
|
8466
|
+
const experimentUrl = `${projectUrl}/experiments/${encodeURIComponent(
|
|
8270
8467
|
await this.name
|
|
8271
8468
|
)}`;
|
|
8272
8469
|
let scores = void 0;
|
|
@@ -8299,7 +8496,7 @@ var Experiment = class extends ObjectFetcher {
|
|
|
8299
8496
|
projectUrl,
|
|
8300
8497
|
experimentUrl,
|
|
8301
8498
|
comparisonExperimentName,
|
|
8302
|
-
scores,
|
|
8499
|
+
scores: scores ?? {},
|
|
8303
8500
|
metrics
|
|
8304
8501
|
};
|
|
8305
8502
|
}
|
|
@@ -8317,10 +8514,21 @@ var Experiment = class extends ObjectFetcher {
|
|
|
8317
8514
|
logFeedback(event) {
|
|
8318
8515
|
logFeedbackImpl(
|
|
8319
8516
|
this.bgLogger,
|
|
8320
|
-
|
|
8517
|
+
this.spanParentObjectType(),
|
|
8518
|
+
this.lazyId,
|
|
8321
8519
|
event
|
|
8322
8520
|
);
|
|
8323
8521
|
}
|
|
8522
|
+
/**
|
|
8523
|
+
* Return a serialized representation of the experiment that can be used to start subspans in other places. See `Span.start_span` for more details.
|
|
8524
|
+
*/
|
|
8525
|
+
async export() {
|
|
8526
|
+
return new SpanParentComponents({
|
|
8527
|
+
objectType: this.spanParentObjectType(),
|
|
8528
|
+
objectId: await this.id,
|
|
8529
|
+
rowId: ""
|
|
8530
|
+
}).toStr();
|
|
8531
|
+
}
|
|
8324
8532
|
/**
|
|
8325
8533
|
* Flush any pending rows to the server.
|
|
8326
8534
|
*/
|
|
@@ -8388,20 +8596,26 @@ var SpanImpl = class _SpanImpl {
|
|
|
8388
8596
|
isMerge;
|
|
8389
8597
|
loggedEndTime;
|
|
8390
8598
|
// For internal use only.
|
|
8391
|
-
|
|
8392
|
-
|
|
8393
|
-
|
|
8394
|
-
|
|
8599
|
+
parentObjectType;
|
|
8600
|
+
parentObjectId;
|
|
8601
|
+
parentRowId;
|
|
8602
|
+
_id;
|
|
8395
8603
|
kind = "span";
|
|
8396
|
-
// root_experiment should only be specified for a root span. parent_span
|
|
8397
|
-
// should only be specified for non-root spans.
|
|
8398
8604
|
constructor(args) {
|
|
8605
|
+
const spanAttributes = args.spanAttributes ?? {};
|
|
8606
|
+
const event = args.event ?? {};
|
|
8607
|
+
const type = args.type ?? (args.parentRowId ? void 0 : args.defaultRootType);
|
|
8399
8608
|
this.loggedEndTime = void 0;
|
|
8609
|
+
this.parentObjectType = args.parentObjectType;
|
|
8610
|
+
this.parentObjectId = args.parentObjectId;
|
|
8611
|
+
this.parentRowId = args.parentRowId;
|
|
8400
8612
|
this.bgLogger = args.bgLogger;
|
|
8401
8613
|
const callerLocation = isomorph_default.getCallerLocation();
|
|
8402
8614
|
const name = (() => {
|
|
8403
8615
|
if (args.name)
|
|
8404
8616
|
return args.name;
|
|
8617
|
+
if (!args.parentRowId)
|
|
8618
|
+
return "root";
|
|
8405
8619
|
if (callerLocation) {
|
|
8406
8620
|
const pathComponents = callerLocation.caller_filename.split("/");
|
|
8407
8621
|
const filename = pathComponents[pathComponents.length - 1];
|
|
@@ -8417,71 +8631,52 @@ var SpanImpl = class _SpanImpl {
|
|
|
8417
8631
|
},
|
|
8418
8632
|
context: { ...callerLocation },
|
|
8419
8633
|
span_attributes: {
|
|
8420
|
-
...args.spanAttributes,
|
|
8421
8634
|
name,
|
|
8635
|
+
type,
|
|
8636
|
+
...spanAttributes,
|
|
8422
8637
|
exec_counter: executionCounter++
|
|
8423
8638
|
},
|
|
8424
8639
|
created: (/* @__PURE__ */ new Date()).toISOString()
|
|
8425
8640
|
};
|
|
8426
|
-
this.
|
|
8427
|
-
this.parentIds = args.parentIds;
|
|
8428
|
-
const id = args.event?.id ?? v4_default();
|
|
8429
|
-
this.rowIds = { id };
|
|
8430
|
-
const parentSpanInfo = "parentSpanInfo" in args ? args.parentSpanInfo : void 0;
|
|
8431
|
-
const parentId = "parentId" in args ? args.parentId : void 0;
|
|
8432
|
-
if (parentSpanInfo && parentId) {
|
|
8433
|
-
throw new Error(
|
|
8434
|
-
"Only one of parentSpanInfo and parentId may be specified"
|
|
8435
|
-
);
|
|
8436
|
-
}
|
|
8437
|
-
if (parentId) {
|
|
8438
|
-
this.rowIds[PARENT_ID_FIELD] = parentId;
|
|
8439
|
-
} else {
|
|
8440
|
-
this.rowIds.span_id = v4_default();
|
|
8441
|
-
if (parentSpanInfo) {
|
|
8442
|
-
this.rowIds.root_span_id = parentSpanInfo.root_span_id;
|
|
8443
|
-
this.internalData.span_parents = [parentSpanInfo.span_id];
|
|
8444
|
-
} else {
|
|
8445
|
-
this.rowIds.root_span_id = this.rowIds.span_id;
|
|
8446
|
-
}
|
|
8447
|
-
}
|
|
8641
|
+
this._id = event.id ?? v4_default();
|
|
8448
8642
|
this.isMerge = false;
|
|
8449
|
-
const { id: _id, ...eventRest } =
|
|
8643
|
+
const { id: _id, ...eventRest } = event;
|
|
8450
8644
|
this.log(eventRest);
|
|
8451
8645
|
this.isMerge = true;
|
|
8452
8646
|
}
|
|
8453
8647
|
get id() {
|
|
8454
|
-
return this.
|
|
8648
|
+
return this._id;
|
|
8455
8649
|
}
|
|
8456
8650
|
log(event) {
|
|
8457
8651
|
const sanitized = validateAndSanitizeExperimentLogPartialArgs(event);
|
|
8458
8652
|
let sanitizedAndInternalData = { ...this.internalData };
|
|
8459
8653
|
mergeDicts(sanitizedAndInternalData, sanitized);
|
|
8460
8654
|
this.internalData = {};
|
|
8461
|
-
if (sanitizedAndInternalData.tags && sanitizedAndInternalData.tags.length > 0 && this.rowIds.span_id !== this.rowIds.root_span_id) {
|
|
8462
|
-
throw new Error("Tags can only be logged to the root span");
|
|
8463
|
-
}
|
|
8464
8655
|
let partialRecord = {
|
|
8465
8656
|
...sanitizedAndInternalData,
|
|
8466
|
-
...this.rowIds,
|
|
8467
8657
|
[IS_MERGE_FIELD]: this.isMerge
|
|
8468
8658
|
};
|
|
8659
|
+
const serializedPartialRecord = JSON.stringify(partialRecord);
|
|
8660
|
+
partialRecord = JSON.parse(serializedPartialRecord);
|
|
8469
8661
|
if (partialRecord.metrics?.end) {
|
|
8470
8662
|
this.loggedEndTime = partialRecord.metrics?.end;
|
|
8471
8663
|
}
|
|
8472
|
-
|
|
8473
|
-
|
|
8474
|
-
|
|
8475
|
-
|
|
8476
|
-
|
|
8477
|
-
|
|
8478
|
-
|
|
8479
|
-
|
|
8664
|
+
if ((partialRecord.tags ?? []).length > 0 && this.parentRowId) {
|
|
8665
|
+
throw new Error("Tags can only be logged to the root span");
|
|
8666
|
+
}
|
|
8667
|
+
const computeRecord = async () => ({
|
|
8668
|
+
...partialRecord,
|
|
8669
|
+
...new SpanParentComponents({
|
|
8670
|
+
objectType: this.parentObjectType,
|
|
8671
|
+
objectId: await this.parentObjectId.get(),
|
|
8672
|
+
rowId: this.parentRowId
|
|
8673
|
+
}).asDict(),
|
|
8674
|
+
id: this.id
|
|
8480
8675
|
});
|
|
8481
|
-
this.bgLogger.log([
|
|
8676
|
+
this.bgLogger.log([new LazyValue(computeRecord)]);
|
|
8482
8677
|
}
|
|
8483
8678
|
logFeedback(event) {
|
|
8484
|
-
logFeedbackImpl(this.bgLogger, this.
|
|
8679
|
+
logFeedbackImpl(this.bgLogger, this.parentObjectType, this.parentObjectId, {
|
|
8485
8680
|
...event,
|
|
8486
8681
|
id: this.id
|
|
8487
8682
|
});
|
|
@@ -8501,25 +8696,15 @@ var SpanImpl = class _SpanImpl {
|
|
|
8501
8696
|
);
|
|
8502
8697
|
}
|
|
8503
8698
|
startSpan(args) {
|
|
8504
|
-
const parentId = args?.parentId ?? (this.rowIds[PARENT_ID_FIELD] ? this.id : void 0);
|
|
8505
|
-
const parentSpanInfo = (() => {
|
|
8506
|
-
if (parentId)
|
|
8507
|
-
return void 0;
|
|
8508
|
-
if (!(this.rowIds.span_id && this.rowIds.root_span_id)) {
|
|
8509
|
-
throw new Error("Impossible");
|
|
8510
|
-
}
|
|
8511
|
-
return {
|
|
8512
|
-
span_id: this.rowIds.span_id,
|
|
8513
|
-
root_span_id: this.rowIds.root_span_id
|
|
8514
|
-
};
|
|
8515
|
-
})();
|
|
8516
8699
|
return new _SpanImpl({
|
|
8517
|
-
|
|
8518
|
-
|
|
8519
|
-
|
|
8520
|
-
|
|
8521
|
-
|
|
8522
|
-
|
|
8700
|
+
...args,
|
|
8701
|
+
...startSpanParentArgs({
|
|
8702
|
+
parent: args?.parent,
|
|
8703
|
+
parentId: args?.parentId ?? (args?.parent ? void 0 : this.id),
|
|
8704
|
+
spanParentObjectType: this.parentObjectType,
|
|
8705
|
+
spanParentObjectId: this.parentObjectId
|
|
8706
|
+
}),
|
|
8707
|
+
bgLogger: this.bgLogger
|
|
8523
8708
|
});
|
|
8524
8709
|
}
|
|
8525
8710
|
end(args) {
|
|
@@ -8533,9 +8718,22 @@ var SpanImpl = class _SpanImpl {
|
|
|
8533
8718
|
this.log({});
|
|
8534
8719
|
return endTime;
|
|
8535
8720
|
}
|
|
8721
|
+
/**
|
|
8722
|
+
* Return a serialized representation of the span that can be used to start subspans in other places. See `Span.start_span` for more details.
|
|
8723
|
+
*/
|
|
8724
|
+
async export() {
|
|
8725
|
+
return new SpanParentComponents({
|
|
8726
|
+
objectType: this.parentObjectType,
|
|
8727
|
+
objectId: await this.parentObjectId.get(),
|
|
8728
|
+
rowId: this.id
|
|
8729
|
+
}).toStr();
|
|
8730
|
+
}
|
|
8536
8731
|
close(args) {
|
|
8537
8732
|
return this.end(args);
|
|
8538
8733
|
}
|
|
8734
|
+
async flush() {
|
|
8735
|
+
return await this.bgLogger.flush();
|
|
8736
|
+
}
|
|
8539
8737
|
};
|
|
8540
8738
|
var Dataset = class extends ObjectFetcher {
|
|
8541
8739
|
lazyMetadata;
|
|
@@ -8622,7 +8820,6 @@ var Dataset = class extends ObjectFetcher {
|
|
|
8622
8820
|
input,
|
|
8623
8821
|
expected: expected === void 0 ? output : expected,
|
|
8624
8822
|
tags,
|
|
8625
|
-
project_id: (await this.project).id,
|
|
8626
8823
|
dataset_id: await this.id,
|
|
8627
8824
|
created: (/* @__PURE__ */ new Date()).toISOString(),
|
|
8628
8825
|
metadata
|
|
@@ -8633,7 +8830,6 @@ var Dataset = class extends ObjectFetcher {
|
|
|
8633
8830
|
delete(id) {
|
|
8634
8831
|
const args = new LazyValue(async () => ({
|
|
8635
8832
|
id,
|
|
8636
|
-
project_id: (await this.project).id,
|
|
8637
8833
|
dataset_id: await this.id,
|
|
8638
8834
|
created: (/* @__PURE__ */ new Date()).toISOString(),
|
|
8639
8835
|
_object_delete: true
|
|
@@ -8651,10 +8847,12 @@ var Dataset = class extends ObjectFetcher {
|
|
|
8651
8847
|
let { summarizeData = true } = options || {};
|
|
8652
8848
|
await this.bgLogger.flush();
|
|
8653
8849
|
const state = await this.getState();
|
|
8654
|
-
const projectUrl = `${state.
|
|
8850
|
+
const projectUrl = `${state.appPublicUrl}/app/${encodeURIComponent(
|
|
8655
8851
|
state.orgName
|
|
8656
8852
|
)}/p/${encodeURIComponent((await this.project).name)}`;
|
|
8657
|
-
const datasetUrl = `${projectUrl}/
|
|
8853
|
+
const datasetUrl = `${projectUrl}/datasets/${encodeURIComponent(
|
|
8854
|
+
await this.name
|
|
8855
|
+
)}`;
|
|
8658
8856
|
let dataSummary = void 0;
|
|
8659
8857
|
if (summarizeData) {
|
|
8660
8858
|
dataSummary = await state.logConn().get_json(
|
|
@@ -8734,7 +8932,7 @@ var Prompt = class {
|
|
|
8734
8932
|
...this.defaults,
|
|
8735
8933
|
...Object.fromEntries(
|
|
8736
8934
|
Object.entries(this.options.params || {}).filter(
|
|
8737
|
-
([k,
|
|
8935
|
+
([k, _v]) => !BRAINTRUST_PARAMS.includes(k)
|
|
8738
8936
|
)
|
|
8739
8937
|
),
|
|
8740
8938
|
...!isEmpty(this.options.model) ? {
|
|
@@ -8768,9 +8966,14 @@ var Prompt = class {
|
|
|
8768
8966
|
"Prompt is a completion prompt. Use buildCompletion() instead"
|
|
8769
8967
|
);
|
|
8770
8968
|
}
|
|
8969
|
+
const render3 = (template) => mustache_default.render(template, buildArgs, void 0, {
|
|
8970
|
+
escape: (v) => typeof v === "string" ? v : JSON.stringify(v)
|
|
8971
|
+
});
|
|
8771
8972
|
const messages = (prompt.messages || []).map((m) => ({
|
|
8772
8973
|
...m,
|
|
8773
|
-
..."content" in m
|
|
8974
|
+
..."content" in m ? {
|
|
8975
|
+
content: typeof m.content === "string" ? render3(m.content) : JSON.parse(render3(JSON.stringify(m.content)))
|
|
8976
|
+
} : {}
|
|
8774
8977
|
}));
|
|
8775
8978
|
return {
|
|
8776
8979
|
...params,
|
|
@@ -8927,7 +9130,7 @@ function wrapBetaChatCompletion(completion) {
|
|
|
8927
9130
|
});
|
|
8928
9131
|
ret.on("chatCompletion", (completion2) => {
|
|
8929
9132
|
span.log({
|
|
8930
|
-
output: completion2.choices
|
|
9133
|
+
output: completion2.choices
|
|
8931
9134
|
});
|
|
8932
9135
|
});
|
|
8933
9136
|
ret.on("end", () => {
|
|
@@ -8955,8 +9158,8 @@ function wrapChatCompletion(completion) {
|
|
|
8955
9158
|
span_info || {}
|
|
8956
9159
|
)
|
|
8957
9160
|
});
|
|
9161
|
+
const startTime = getCurrentUnixTimestamp();
|
|
8958
9162
|
if (params.stream) {
|
|
8959
|
-
const startTime = getCurrentUnixTimestamp();
|
|
8960
9163
|
const ret = await completion(
|
|
8961
9164
|
params,
|
|
8962
9165
|
options
|
|
@@ -8976,8 +9179,9 @@ function wrapChatCompletion(completion) {
|
|
|
8976
9179
|
metadata: {
|
|
8977
9180
|
...rest2
|
|
8978
9181
|
},
|
|
8979
|
-
output: ret.choices
|
|
9182
|
+
output: ret.choices,
|
|
8980
9183
|
metrics: {
|
|
9184
|
+
time_to_first_token: getCurrentUnixTimestamp() - startTime,
|
|
8981
9185
|
tokens: ret.usage?.total_tokens,
|
|
8982
9186
|
prompt_tokens: ret.usage?.prompt_tokens,
|
|
8983
9187
|
completion_tokens: ret.usage?.completion_tokens
|
|
@@ -9027,6 +9231,52 @@ function wrapEmbeddings(create) {
|
|
|
9027
9231
|
);
|
|
9028
9232
|
};
|
|
9029
9233
|
}
|
|
9234
|
+
function postprocessStreamingResults(allResults) {
|
|
9235
|
+
let role = void 0;
|
|
9236
|
+
let content = void 0;
|
|
9237
|
+
let tool_calls = void 0;
|
|
9238
|
+
let finish_reason = void 0;
|
|
9239
|
+
for (const result of allResults) {
|
|
9240
|
+
const delta = result.choices?.[0]?.delta;
|
|
9241
|
+
if (!delta) {
|
|
9242
|
+
continue;
|
|
9243
|
+
}
|
|
9244
|
+
if (!role && delta.role) {
|
|
9245
|
+
role = delta.role;
|
|
9246
|
+
}
|
|
9247
|
+
if (delta.finish_reason) {
|
|
9248
|
+
finish_reason = delta.finish_reason;
|
|
9249
|
+
}
|
|
9250
|
+
if (delta.content) {
|
|
9251
|
+
content = (content || "") + delta.content;
|
|
9252
|
+
}
|
|
9253
|
+
if (delta.tool_calls) {
|
|
9254
|
+
if (!tool_calls) {
|
|
9255
|
+
tool_calls = [
|
|
9256
|
+
{
|
|
9257
|
+
id: delta.tool_calls[0].id,
|
|
9258
|
+
type: delta.tool_calls[0].type,
|
|
9259
|
+
function: delta.tool_calls[0].function
|
|
9260
|
+
}
|
|
9261
|
+
];
|
|
9262
|
+
} else {
|
|
9263
|
+
tool_calls[0].function.arguments += delta.tool_calls[0].function.arguments;
|
|
9264
|
+
}
|
|
9265
|
+
}
|
|
9266
|
+
}
|
|
9267
|
+
return [
|
|
9268
|
+
{
|
|
9269
|
+
index: 0,
|
|
9270
|
+
message: {
|
|
9271
|
+
role,
|
|
9272
|
+
content,
|
|
9273
|
+
tool_calls
|
|
9274
|
+
},
|
|
9275
|
+
logprobs: null,
|
|
9276
|
+
finish_reason
|
|
9277
|
+
}
|
|
9278
|
+
];
|
|
9279
|
+
}
|
|
9030
9280
|
var WrapperStream = class {
|
|
9031
9281
|
span;
|
|
9032
9282
|
iter;
|
|
@@ -9054,7 +9304,7 @@ var WrapperStream = class {
|
|
|
9054
9304
|
yield item;
|
|
9055
9305
|
}
|
|
9056
9306
|
this.span.log({
|
|
9057
|
-
output: allResults
|
|
9307
|
+
output: postprocessStreamingResults(allResults)
|
|
9058
9308
|
});
|
|
9059
9309
|
} finally {
|
|
9060
9310
|
this.span.end();
|
|
@@ -9065,6 +9315,7 @@ var WrapperStream = class {
|
|
|
9065
9315
|
// src/browser.ts
|
|
9066
9316
|
configureBrowser();
|
|
9067
9317
|
export {
|
|
9318
|
+
Dataset,
|
|
9068
9319
|
Experiment,
|
|
9069
9320
|
Logger,
|
|
9070
9321
|
NOOP_SPAN,
|