braintrust 0.0.93 → 0.0.95
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser.js +380 -266
- package/dist/cli.js +328 -317
- package/dist/framework.d.ts +21 -8
- package/dist/index.d.ts +1 -1
- package/dist/index.js +435 -273
- package/dist/logger.d.ts +57 -51
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/dist/util.d.ts +6 -0
- package/package.json +2 -2
package/dist/browser.js
CHANGED
|
@@ -183,6 +183,21 @@ function getCurrentUnixTimestamp() {
|
|
|
183
183
|
function isEmpty(a) {
|
|
184
184
|
return a === void 0 || a === null;
|
|
185
185
|
}
|
|
186
|
+
var LazyValue = class {
|
|
187
|
+
constructor(callable) {
|
|
188
|
+
this.value = {
|
|
189
|
+
hasComputed: false
|
|
190
|
+
};
|
|
191
|
+
this.callable = callable;
|
|
192
|
+
}
|
|
193
|
+
async get() {
|
|
194
|
+
if (this.value.hasComputed) {
|
|
195
|
+
return this.value.val;
|
|
196
|
+
}
|
|
197
|
+
this.value = { hasComputed: true, val: await this.callable() };
|
|
198
|
+
return this.value.val;
|
|
199
|
+
}
|
|
200
|
+
};
|
|
186
201
|
|
|
187
202
|
// src/logger.ts
|
|
188
203
|
var NoopSpan = class {
|
|
@@ -397,25 +412,25 @@ function logFeedbackImpl(bgLogger, parentIds, {
|
|
|
397
412
|
updateEvent = Object.fromEntries(
|
|
398
413
|
Object.entries(updateEvent).filter(([_, v]) => !isEmpty(v))
|
|
399
414
|
);
|
|
400
|
-
const trueParentIds = (async () => {
|
|
401
|
-
const { kind, ...ids } = await parentIds;
|
|
415
|
+
const trueParentIds = new LazyValue(async () => {
|
|
416
|
+
const { kind, ...ids } = await parentIds.get();
|
|
402
417
|
return ids;
|
|
403
|
-
})
|
|
418
|
+
});
|
|
404
419
|
if (Object.keys(updateEvent).length > 0) {
|
|
405
|
-
const record = (async () => {
|
|
420
|
+
const record = new LazyValue(async () => {
|
|
406
421
|
return {
|
|
407
422
|
id,
|
|
408
423
|
...updateEvent,
|
|
409
|
-
...await trueParentIds,
|
|
424
|
+
...await trueParentIds.get(),
|
|
410
425
|
[AUDIT_SOURCE_FIELD]: source,
|
|
411
426
|
[AUDIT_METADATA_FIELD]: metadata,
|
|
412
427
|
[IS_MERGE_FIELD]: true
|
|
413
428
|
};
|
|
414
|
-
})
|
|
429
|
+
});
|
|
415
430
|
bgLogger.log([record]);
|
|
416
431
|
}
|
|
417
432
|
if (!isEmpty(comment)) {
|
|
418
|
-
const record = (async () => {
|
|
433
|
+
const record = new LazyValue(async () => {
|
|
419
434
|
return {
|
|
420
435
|
id: v4_default(),
|
|
421
436
|
created: (/* @__PURE__ */ new Date()).toISOString(),
|
|
@@ -427,11 +442,11 @@ function logFeedbackImpl(bgLogger, parentIds, {
|
|
|
427
442
|
comment: {
|
|
428
443
|
text: comment
|
|
429
444
|
},
|
|
430
|
-
...await trueParentIds,
|
|
445
|
+
...await trueParentIds.get(),
|
|
431
446
|
[AUDIT_SOURCE_FIELD]: source,
|
|
432
447
|
[AUDIT_METADATA_FIELD]: metadata
|
|
433
448
|
};
|
|
434
|
-
})
|
|
449
|
+
});
|
|
435
450
|
bgLogger.log([record]);
|
|
436
451
|
}
|
|
437
452
|
}
|
|
@@ -441,22 +456,24 @@ var Logger = class {
|
|
|
441
456
|
this.kind = "logger";
|
|
442
457
|
this.lazyMetadata = lazyMetadata;
|
|
443
458
|
this.logOptions = logOptions;
|
|
444
|
-
const logConn =
|
|
459
|
+
const logConn = new LazyValue(
|
|
460
|
+
() => this.getState().then((state) => state.logConn())
|
|
461
|
+
);
|
|
445
462
|
this.bgLogger = new BackgroundLogger(logConn);
|
|
446
463
|
this.lastStartTime = getCurrentUnixTimestamp();
|
|
447
464
|
}
|
|
448
465
|
get org_id() {
|
|
449
466
|
return (async () => {
|
|
450
|
-
return (await this.lazyMetadata).org_id;
|
|
467
|
+
return (await this.lazyMetadata.get()).org_id;
|
|
451
468
|
})();
|
|
452
469
|
}
|
|
453
470
|
get project() {
|
|
454
471
|
return (async () => {
|
|
455
|
-
return (await this.lazyMetadata).project;
|
|
472
|
+
return (await this.lazyMetadata.get()).project;
|
|
456
473
|
})();
|
|
457
474
|
}
|
|
458
475
|
async getState() {
|
|
459
|
-
await this.lazyMetadata;
|
|
476
|
+
await this.lazyMetadata.get();
|
|
460
477
|
return _state;
|
|
461
478
|
}
|
|
462
479
|
/**
|
|
@@ -531,7 +548,7 @@ var Logger = class {
|
|
|
531
548
|
startSpan(args) {
|
|
532
549
|
const { name, ...argsRest } = args ?? {};
|
|
533
550
|
return new SpanImpl({
|
|
534
|
-
parentIds: this.lazyParentIds(),
|
|
551
|
+
parentIds: new LazyValue(() => this.lazyParentIds()),
|
|
535
552
|
bgLogger: this.bgLogger,
|
|
536
553
|
name: name ?? "root",
|
|
537
554
|
...argsRest
|
|
@@ -549,7 +566,11 @@ var Logger = class {
|
|
|
549
566
|
* @param event.source (Optional) the source of the feedback. Must be one of "external" (default), "app", or "api".
|
|
550
567
|
*/
|
|
551
568
|
logFeedback(event) {
|
|
552
|
-
logFeedbackImpl(
|
|
569
|
+
logFeedbackImpl(
|
|
570
|
+
this.bgLogger,
|
|
571
|
+
new LazyValue(() => this.lazyParentIds()),
|
|
572
|
+
event
|
|
573
|
+
);
|
|
553
574
|
}
|
|
554
575
|
/*
|
|
555
576
|
* Flush any pending logs to the server.
|
|
@@ -599,9 +620,20 @@ var BackgroundLogger = class {
|
|
|
599
620
|
}
|
|
600
621
|
async flush_once(batchSize = DefaultBatchSize) {
|
|
601
622
|
this.active_flush_resolved = false;
|
|
602
|
-
const
|
|
623
|
+
const itemLazyValues = this.items;
|
|
603
624
|
this.items = [];
|
|
604
|
-
const allItems =
|
|
625
|
+
const allItems = await (async () => {
|
|
626
|
+
try {
|
|
627
|
+
const itemPromises = itemLazyValues.map((x) => x.get());
|
|
628
|
+
return mergeRowBatch(await Promise.all(itemPromises)).reverse();
|
|
629
|
+
} catch (e) {
|
|
630
|
+
console.warn(
|
|
631
|
+
"Encountered error when constructing records to flush:\n",
|
|
632
|
+
e
|
|
633
|
+
);
|
|
634
|
+
return [];
|
|
635
|
+
}
|
|
636
|
+
})();
|
|
605
637
|
let postPromises = [];
|
|
606
638
|
while (true) {
|
|
607
639
|
const items = [];
|
|
@@ -626,9 +658,7 @@ var BackgroundLogger = class {
|
|
|
626
658
|
for (let i = 0; i < NumRetries; i++) {
|
|
627
659
|
const startTime = now();
|
|
628
660
|
try {
|
|
629
|
-
return (await (await this.logConn).post_json("logs", itemsS)).map(
|
|
630
|
-
(res) => res.id
|
|
631
|
-
);
|
|
661
|
+
return (await (await this.logConn.get()).post_json("logs", itemsS)).map((res) => res.id);
|
|
632
662
|
} catch (e) {
|
|
633
663
|
const retryingText = i + 1 === NumRetries ? "" : " Retrying";
|
|
634
664
|
const errMsg = (() => {
|
|
@@ -674,6 +704,7 @@ function init(project, options = {}) {
|
|
|
674
704
|
dataset,
|
|
675
705
|
baseExperiment,
|
|
676
706
|
isPublic,
|
|
707
|
+
open,
|
|
677
708
|
update,
|
|
678
709
|
appUrl,
|
|
679
710
|
apiKey,
|
|
@@ -681,82 +712,121 @@ function init(project, options = {}) {
|
|
|
681
712
|
metadata,
|
|
682
713
|
gitMetadataSettings
|
|
683
714
|
} = options || {};
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
apiKey,
|
|
688
|
-
appUrl
|
|
689
|
-
});
|
|
690
|
-
const args = {
|
|
691
|
-
project_name: project,
|
|
692
|
-
org_id: _state.orgId
|
|
693
|
-
};
|
|
694
|
-
if (experiment) {
|
|
695
|
-
args["experiment_name"] = experiment;
|
|
696
|
-
}
|
|
697
|
-
if (description) {
|
|
698
|
-
args["description"] = description;
|
|
715
|
+
if (open) {
|
|
716
|
+
if (isEmpty(experiment)) {
|
|
717
|
+
throw new Error("Cannot open an experiment without specifying its name");
|
|
699
718
|
}
|
|
700
719
|
if (update) {
|
|
701
|
-
|
|
720
|
+
throw new Error("Cannot open and update an experiment at the same time");
|
|
702
721
|
}
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
722
|
+
const lazyMetadata2 = new LazyValue(async () => {
|
|
723
|
+
await login({
|
|
724
|
+
orgName,
|
|
725
|
+
apiKey,
|
|
726
|
+
appUrl
|
|
727
|
+
});
|
|
728
|
+
const args = {
|
|
729
|
+
project_name: project,
|
|
730
|
+
org_name: _state.orgName,
|
|
731
|
+
experiment_name: experiment
|
|
732
|
+
};
|
|
733
|
+
const response = await _state.apiConn().post_json("api/experiment/get", args);
|
|
734
|
+
if (response.length === 0) {
|
|
735
|
+
throw new Error(
|
|
736
|
+
`Experiment ${experiment} not found in project ${project}.`
|
|
737
|
+
);
|
|
706
738
|
}
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
}
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
args
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
delete args["base_experiment"];
|
|
742
|
-
} else {
|
|
743
|
-
throw e;
|
|
739
|
+
const info = response[0];
|
|
740
|
+
return {
|
|
741
|
+
id: info.id,
|
|
742
|
+
name: info.name,
|
|
743
|
+
fullInfo: info
|
|
744
|
+
};
|
|
745
|
+
});
|
|
746
|
+
return new ReadonlyExperiment(
|
|
747
|
+
lazyMetadata2
|
|
748
|
+
);
|
|
749
|
+
}
|
|
750
|
+
const lazyMetadata = new LazyValue(
|
|
751
|
+
async () => {
|
|
752
|
+
await login({
|
|
753
|
+
orgName,
|
|
754
|
+
apiKey,
|
|
755
|
+
appUrl
|
|
756
|
+
});
|
|
757
|
+
const args = {
|
|
758
|
+
project_name: project,
|
|
759
|
+
org_id: _state.orgId
|
|
760
|
+
};
|
|
761
|
+
if (experiment) {
|
|
762
|
+
args["experiment_name"] = experiment;
|
|
763
|
+
}
|
|
764
|
+
if (description) {
|
|
765
|
+
args["description"] = description;
|
|
766
|
+
}
|
|
767
|
+
if (update) {
|
|
768
|
+
args["update"] = update;
|
|
769
|
+
}
|
|
770
|
+
let mergedGitMetadataSettings = {
|
|
771
|
+
..._state.gitMetadataSettings || {
|
|
772
|
+
collect: "all"
|
|
744
773
|
}
|
|
774
|
+
};
|
|
775
|
+
if (gitMetadataSettings) {
|
|
776
|
+
mergedGitMetadataSettings = mergeGitMetadataSettings(
|
|
777
|
+
mergedGitMetadataSettings,
|
|
778
|
+
gitMetadataSettings
|
|
779
|
+
);
|
|
745
780
|
}
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
id: response.project.id,
|
|
750
|
-
name: response.project.name,
|
|
751
|
-
fullInfo: response.project
|
|
752
|
-
},
|
|
753
|
-
experiment: {
|
|
754
|
-
id: response.experiment.id,
|
|
755
|
-
name: response.experiment.name,
|
|
756
|
-
fullInfo: response.experiment
|
|
781
|
+
const repoStatus = await isomorph_default.getRepoStatus(gitMetadataSettings);
|
|
782
|
+
if (repoStatus) {
|
|
783
|
+
args["repo_info"] = repoStatus;
|
|
757
784
|
}
|
|
758
|
-
|
|
759
|
-
|
|
785
|
+
if (baseExperiment) {
|
|
786
|
+
args["base_experiment"] = baseExperiment;
|
|
787
|
+
} else {
|
|
788
|
+
args["ancestor_commits"] = await isomorph_default.getPastNAncestors();
|
|
789
|
+
}
|
|
790
|
+
if (dataset !== void 0) {
|
|
791
|
+
args["dataset_id"] = await dataset.id;
|
|
792
|
+
args["dataset_version"] = await dataset.version();
|
|
793
|
+
}
|
|
794
|
+
if (isPublic !== void 0) {
|
|
795
|
+
args["public"] = isPublic;
|
|
796
|
+
}
|
|
797
|
+
if (metadata) {
|
|
798
|
+
args["metadata"] = metadata;
|
|
799
|
+
}
|
|
800
|
+
let response = null;
|
|
801
|
+
while (true) {
|
|
802
|
+
try {
|
|
803
|
+
response = await _state.apiConn().post_json("api/experiment/register", args);
|
|
804
|
+
break;
|
|
805
|
+
} catch (e) {
|
|
806
|
+
if (args["base_experiment"] && `${"data" in e && e.data}`.includes("base experiment")) {
|
|
807
|
+
console.warn(
|
|
808
|
+
`Base experiment ${args["base_experiment"]} not found.`
|
|
809
|
+
);
|
|
810
|
+
delete args["base_experiment"];
|
|
811
|
+
} else {
|
|
812
|
+
throw e;
|
|
813
|
+
}
|
|
814
|
+
}
|
|
815
|
+
}
|
|
816
|
+
return {
|
|
817
|
+
project: {
|
|
818
|
+
id: response.project.id,
|
|
819
|
+
name: response.project.name,
|
|
820
|
+
fullInfo: response.project
|
|
821
|
+
},
|
|
822
|
+
experiment: {
|
|
823
|
+
id: response.experiment.id,
|
|
824
|
+
name: response.experiment.name,
|
|
825
|
+
fullInfo: response.experiment
|
|
826
|
+
}
|
|
827
|
+
};
|
|
828
|
+
}
|
|
829
|
+
);
|
|
760
830
|
const ret = new Experiment(lazyMetadata, dataset);
|
|
761
831
|
if (options.setCurrent ?? true) {
|
|
762
832
|
_state.currentExperiment = ret;
|
|
@@ -779,32 +849,34 @@ function withLogger(callback, options = {}) {
|
|
|
779
849
|
}
|
|
780
850
|
function initDataset(project, options = {}) {
|
|
781
851
|
const { dataset, description, version, appUrl, apiKey, orgName } = options || {};
|
|
782
|
-
const lazyMetadata =
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
852
|
+
const lazyMetadata = new LazyValue(
|
|
853
|
+
async () => {
|
|
854
|
+
await login({
|
|
855
|
+
orgName,
|
|
856
|
+
apiKey,
|
|
857
|
+
appUrl
|
|
858
|
+
});
|
|
859
|
+
const args = {
|
|
860
|
+
org_id: _state.orgId,
|
|
861
|
+
project_name: project,
|
|
862
|
+
dataset_name: dataset,
|
|
863
|
+
description
|
|
864
|
+
};
|
|
865
|
+
const response = await _state.apiConn().post_json("api/dataset/register", args);
|
|
866
|
+
return {
|
|
867
|
+
project: {
|
|
868
|
+
id: response.project.id,
|
|
869
|
+
name: response.project.name,
|
|
870
|
+
fullInfo: response.project
|
|
871
|
+
},
|
|
872
|
+
dataset: {
|
|
873
|
+
id: response.dataset.id,
|
|
874
|
+
name: response.dataset.name,
|
|
875
|
+
fullInfo: response.dataset
|
|
876
|
+
}
|
|
877
|
+
};
|
|
878
|
+
}
|
|
879
|
+
);
|
|
808
880
|
return new Dataset(lazyMetadata, version);
|
|
809
881
|
}
|
|
810
882
|
function withDataset(project, callback, options = {}) {
|
|
@@ -824,46 +896,48 @@ function initLogger(options = {}) {
|
|
|
824
896
|
orgName,
|
|
825
897
|
forceLogin
|
|
826
898
|
} = options || {};
|
|
827
|
-
const lazyMetadata =
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
const org_id = _state.orgId;
|
|
835
|
-
if (projectId === void 0) {
|
|
836
|
-
const response = await _state.apiConn().post_json("api/project/register", {
|
|
837
|
-
project_name: projectName || GLOBAL_PROJECT,
|
|
838
|
-
org_id
|
|
899
|
+
const lazyMetadata = new LazyValue(
|
|
900
|
+
async () => {
|
|
901
|
+
await login({
|
|
902
|
+
orgName,
|
|
903
|
+
apiKey,
|
|
904
|
+
appUrl,
|
|
905
|
+
forceLogin
|
|
839
906
|
});
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
project
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
907
|
+
const org_id = _state.orgId;
|
|
908
|
+
if (projectId === void 0) {
|
|
909
|
+
const response = await _state.apiConn().post_json("api/project/register", {
|
|
910
|
+
project_name: projectName || GLOBAL_PROJECT,
|
|
911
|
+
org_id
|
|
912
|
+
});
|
|
913
|
+
return {
|
|
914
|
+
org_id,
|
|
915
|
+
project: {
|
|
916
|
+
id: response.project.id,
|
|
917
|
+
name: response.project.name,
|
|
918
|
+
fullInfo: response.project
|
|
919
|
+
}
|
|
920
|
+
};
|
|
921
|
+
} else if (projectName === void 0) {
|
|
922
|
+
const response = await _state.apiConn().get_json("api/project", {
|
|
923
|
+
id: projectId
|
|
924
|
+
});
|
|
925
|
+
return {
|
|
926
|
+
org_id,
|
|
927
|
+
project: {
|
|
928
|
+
id: projectId,
|
|
929
|
+
name: response.name,
|
|
930
|
+
fullInfo: response.project
|
|
931
|
+
}
|
|
932
|
+
};
|
|
933
|
+
} else {
|
|
934
|
+
return {
|
|
935
|
+
org_id,
|
|
936
|
+
project: { id: projectId, name: projectName, fullInfo: {} }
|
|
937
|
+
};
|
|
938
|
+
}
|
|
865
939
|
}
|
|
866
|
-
|
|
940
|
+
);
|
|
867
941
|
const ret = new Logger(lazyMetadata, {
|
|
868
942
|
asyncFlush
|
|
869
943
|
});
|
|
@@ -1069,15 +1143,15 @@ function validateAndSanitizeExperimentLogPartialArgs(event) {
|
|
|
1069
1143
|
}
|
|
1070
1144
|
}
|
|
1071
1145
|
function validateAndSanitizeExperimentLogFullArgs(event, hasDataset) {
|
|
1072
|
-
if ("input" in event && event.input && "inputs" in event && event.inputs || !("input" in event) && !("inputs" in event)) {
|
|
1146
|
+
if ("input" in event && !isEmpty(event.input) && "inputs" in event && !isEmpty(event.inputs) || !("input" in event) && !("inputs" in event)) {
|
|
1073
1147
|
throw new Error(
|
|
1074
1148
|
"Exactly one of input or inputs (deprecated) must be specified. Prefer input."
|
|
1075
1149
|
);
|
|
1076
1150
|
}
|
|
1077
|
-
if (
|
|
1151
|
+
if (isEmpty(event.output)) {
|
|
1078
1152
|
throw new Error("output must be specified");
|
|
1079
1153
|
}
|
|
1080
|
-
if (
|
|
1154
|
+
if (isEmpty(event.scores)) {
|
|
1081
1155
|
throw new Error("scores must be specified");
|
|
1082
1156
|
}
|
|
1083
1157
|
if (hasDataset && event.datasetRecordId === void 0) {
|
|
@@ -1089,33 +1163,88 @@ function validateAndSanitizeExperimentLogFullArgs(event, hasDataset) {
|
|
|
1089
1163
|
}
|
|
1090
1164
|
return event;
|
|
1091
1165
|
}
|
|
1092
|
-
var
|
|
1166
|
+
var ObjectFetcher = class {
|
|
1167
|
+
constructor(objectType, pinnedVersion) {
|
|
1168
|
+
this.objectType = objectType;
|
|
1169
|
+
this.pinnedVersion = pinnedVersion;
|
|
1170
|
+
this._fetchedData = void 0;
|
|
1171
|
+
}
|
|
1172
|
+
get id() {
|
|
1173
|
+
throw new Error("ObjectFetcher subclasses must have an 'id' attribute");
|
|
1174
|
+
}
|
|
1175
|
+
async getState() {
|
|
1176
|
+
throw new Error("ObjectFetcher subclasses must have a 'getState' method");
|
|
1177
|
+
}
|
|
1178
|
+
async *fetch() {
|
|
1179
|
+
const records = await this.fetchedData();
|
|
1180
|
+
for (const record of records) {
|
|
1181
|
+
yield record;
|
|
1182
|
+
}
|
|
1183
|
+
}
|
|
1184
|
+
[Symbol.iterator]() {
|
|
1185
|
+
return this.fetch();
|
|
1186
|
+
}
|
|
1187
|
+
async fetchedData() {
|
|
1188
|
+
if (this._fetchedData === void 0) {
|
|
1189
|
+
const state = await this.getState();
|
|
1190
|
+
const resp = await state.logConn().get(`object/${this.objectType}`, {
|
|
1191
|
+
id: await this.id,
|
|
1192
|
+
fmt: "json2",
|
|
1193
|
+
version: this.pinnedVersion
|
|
1194
|
+
});
|
|
1195
|
+
this._fetchedData = await resp.json();
|
|
1196
|
+
}
|
|
1197
|
+
return this._fetchedData || [];
|
|
1198
|
+
}
|
|
1199
|
+
clearCache() {
|
|
1200
|
+
this._fetchedData = void 0;
|
|
1201
|
+
}
|
|
1202
|
+
async version() {
|
|
1203
|
+
if (this.pinnedVersion !== void 0) {
|
|
1204
|
+
return this.pinnedVersion;
|
|
1205
|
+
} else {
|
|
1206
|
+
const fetchedData = await this.fetchedData();
|
|
1207
|
+
let maxVersion = void 0;
|
|
1208
|
+
for (const record of fetchedData) {
|
|
1209
|
+
const xactId = record[TRANSACTION_ID_FIELD];
|
|
1210
|
+
if (maxVersion === void 0 || (xactId ?? xactId > maxVersion)) {
|
|
1211
|
+
maxVersion = xactId;
|
|
1212
|
+
}
|
|
1213
|
+
}
|
|
1214
|
+
return maxVersion;
|
|
1215
|
+
}
|
|
1216
|
+
}
|
|
1217
|
+
};
|
|
1218
|
+
var Experiment = class extends ObjectFetcher {
|
|
1093
1219
|
constructor(lazyMetadata, dataset) {
|
|
1220
|
+
super("experiment", void 0);
|
|
1094
1221
|
// For type identification.
|
|
1095
1222
|
this.kind = "experiment";
|
|
1096
1223
|
this.lazyMetadata = lazyMetadata;
|
|
1097
1224
|
this.dataset = dataset;
|
|
1098
|
-
const logConn =
|
|
1225
|
+
const logConn = new LazyValue(
|
|
1226
|
+
() => this.getState().then((state) => state.logConn())
|
|
1227
|
+
);
|
|
1099
1228
|
this.bgLogger = new BackgroundLogger(logConn);
|
|
1100
1229
|
this.lastStartTime = getCurrentUnixTimestamp();
|
|
1101
1230
|
}
|
|
1102
1231
|
get id() {
|
|
1103
1232
|
return (async () => {
|
|
1104
|
-
return (await this.lazyMetadata).experiment.id;
|
|
1233
|
+
return (await this.lazyMetadata.get()).experiment.id;
|
|
1105
1234
|
})();
|
|
1106
1235
|
}
|
|
1107
1236
|
get name() {
|
|
1108
1237
|
return (async () => {
|
|
1109
|
-
return (await this.lazyMetadata).experiment.name;
|
|
1238
|
+
return (await this.lazyMetadata.get()).experiment.name;
|
|
1110
1239
|
})();
|
|
1111
1240
|
}
|
|
1112
1241
|
get project() {
|
|
1113
1242
|
return (async () => {
|
|
1114
|
-
return (await this.lazyMetadata).project;
|
|
1243
|
+
return (await this.lazyMetadata.get()).project;
|
|
1115
1244
|
})();
|
|
1116
1245
|
}
|
|
1117
1246
|
async getState() {
|
|
1118
|
-
await this.lazyMetadata;
|
|
1247
|
+
await this.lazyMetadata.get();
|
|
1119
1248
|
return _state;
|
|
1120
1249
|
}
|
|
1121
1250
|
/**
|
|
@@ -1175,12 +1304,32 @@ var Experiment = class {
|
|
|
1175
1304
|
startSpan(args) {
|
|
1176
1305
|
const { name, ...argsRest } = args ?? {};
|
|
1177
1306
|
return new SpanImpl({
|
|
1178
|
-
parentIds: this.lazyParentIds(),
|
|
1307
|
+
parentIds: new LazyValue(() => this.lazyParentIds()),
|
|
1179
1308
|
bgLogger: this.bgLogger,
|
|
1180
1309
|
name: name ?? "root",
|
|
1181
1310
|
...argsRest
|
|
1182
1311
|
});
|
|
1183
1312
|
}
|
|
1313
|
+
async fetchBaseExperiment() {
|
|
1314
|
+
const state = await this.getState();
|
|
1315
|
+
const conn = state.apiConn();
|
|
1316
|
+
try {
|
|
1317
|
+
const resp = await conn.post("/api/base_experiment/get_id", {
|
|
1318
|
+
id: await this.id
|
|
1319
|
+
});
|
|
1320
|
+
const base = await resp.json();
|
|
1321
|
+
return {
|
|
1322
|
+
id: base["base_exp_id"],
|
|
1323
|
+
name: base["base_exp_name"]
|
|
1324
|
+
};
|
|
1325
|
+
} catch (e) {
|
|
1326
|
+
if (e instanceof FailedHTTPResponse && e.status === 400) {
|
|
1327
|
+
return null;
|
|
1328
|
+
} else {
|
|
1329
|
+
throw e;
|
|
1330
|
+
}
|
|
1331
|
+
}
|
|
1332
|
+
}
|
|
1184
1333
|
/**
|
|
1185
1334
|
* Summarize the experiment, including the scores (compared to the closest reference experiment) and metadata.
|
|
1186
1335
|
*
|
|
@@ -1204,14 +1353,10 @@ var Experiment = class {
|
|
|
1204
1353
|
let comparisonExperimentName = void 0;
|
|
1205
1354
|
if (summarizeScores) {
|
|
1206
1355
|
if (comparisonExperimentId === void 0) {
|
|
1207
|
-
const
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
const base_experiments = await resp.json();
|
|
1212
|
-
if (base_experiments.length > 0) {
|
|
1213
|
-
comparisonExperimentId = base_experiments[0]["base_exp_id"];
|
|
1214
|
-
comparisonExperimentName = base_experiments[0]["base_exp_name"];
|
|
1356
|
+
const baseExperiment = await this.fetchBaseExperiment();
|
|
1357
|
+
if (baseExperiment !== null) {
|
|
1358
|
+
comparisonExperimentId = baseExperiment.id;
|
|
1359
|
+
comparisonExperimentName = baseExperiment.name;
|
|
1215
1360
|
}
|
|
1216
1361
|
}
|
|
1217
1362
|
if (comparisonExperimentId !== void 0) {
|
|
@@ -1249,7 +1394,11 @@ var Experiment = class {
|
|
|
1249
1394
|
* @param event.source (Optional) the source of the feedback. Must be one of "external" (default), "app", or "api".
|
|
1250
1395
|
*/
|
|
1251
1396
|
logFeedback(event) {
|
|
1252
|
-
logFeedbackImpl(
|
|
1397
|
+
logFeedbackImpl(
|
|
1398
|
+
this.bgLogger,
|
|
1399
|
+
new LazyValue(() => this.lazyParentIds()),
|
|
1400
|
+
event
|
|
1401
|
+
);
|
|
1253
1402
|
}
|
|
1254
1403
|
/**
|
|
1255
1404
|
* Flush any pending rows to the server.
|
|
@@ -1267,6 +1416,40 @@ var Experiment = class {
|
|
|
1267
1416
|
return this.id;
|
|
1268
1417
|
}
|
|
1269
1418
|
};
|
|
1419
|
+
var ReadonlyExperiment = class extends ObjectFetcher {
|
|
1420
|
+
constructor(lazyMetadata) {
|
|
1421
|
+
super("experiment", void 0);
|
|
1422
|
+
this.lazyMetadata = lazyMetadata;
|
|
1423
|
+
}
|
|
1424
|
+
get id() {
|
|
1425
|
+
return (async () => {
|
|
1426
|
+
return (await this.lazyMetadata.get()).id;
|
|
1427
|
+
})();
|
|
1428
|
+
}
|
|
1429
|
+
get name() {
|
|
1430
|
+
return (async () => {
|
|
1431
|
+
return (await this.lazyMetadata.get()).name;
|
|
1432
|
+
})();
|
|
1433
|
+
}
|
|
1434
|
+
async getState() {
|
|
1435
|
+
await this.lazyMetadata.get();
|
|
1436
|
+
return _state;
|
|
1437
|
+
}
|
|
1438
|
+
async *asDataset() {
|
|
1439
|
+
const records = this.fetch();
|
|
1440
|
+
for await (const record of records) {
|
|
1441
|
+
if (record.root_span_id !== record.span_id) {
|
|
1442
|
+
continue;
|
|
1443
|
+
}
|
|
1444
|
+
const { output, expected } = record;
|
|
1445
|
+
yield {
|
|
1446
|
+
input: record.input,
|
|
1447
|
+
expected: expected ?? output
|
|
1448
|
+
};
|
|
1449
|
+
}
|
|
1450
|
+
}
|
|
1451
|
+
};
|
|
1452
|
+
var executionCounter = 0;
|
|
1270
1453
|
var SpanImpl = class _SpanImpl {
|
|
1271
1454
|
// root_experiment should only be specified for a root span. parent_span
|
|
1272
1455
|
// should only be specified for non-root spans.
|
|
@@ -1292,7 +1475,11 @@ var SpanImpl = class _SpanImpl {
|
|
|
1292
1475
|
start: args.startTime ?? getCurrentUnixTimestamp()
|
|
1293
1476
|
},
|
|
1294
1477
|
context: { ...callerLocation },
|
|
1295
|
-
span_attributes: {
|
|
1478
|
+
span_attributes: {
|
|
1479
|
+
...args.spanAttributes,
|
|
1480
|
+
name,
|
|
1481
|
+
exec_counter: executionCounter++
|
|
1482
|
+
},
|
|
1296
1483
|
created: (/* @__PURE__ */ new Date()).toISOString()
|
|
1297
1484
|
};
|
|
1298
1485
|
this.parentIds = args.parentIds;
|
|
@@ -1330,18 +1517,18 @@ var SpanImpl = class _SpanImpl {
|
|
|
1330
1517
|
if (sanitizedAndInternalData.metrics?.end) {
|
|
1331
1518
|
this.loggedEndTime = sanitizedAndInternalData.metrics?.end;
|
|
1332
1519
|
}
|
|
1333
|
-
const parentIds = (async () => {
|
|
1334
|
-
const { kind, ...ids } = await this.parentIds;
|
|
1520
|
+
const parentIds = new LazyValue(async () => {
|
|
1521
|
+
const { kind, ...ids } = await this.parentIds.get();
|
|
1335
1522
|
return ids;
|
|
1336
|
-
})
|
|
1337
|
-
const record = (async () => {
|
|
1523
|
+
});
|
|
1524
|
+
const record = new LazyValue(async () => {
|
|
1338
1525
|
return {
|
|
1339
1526
|
...sanitizedAndInternalData,
|
|
1340
1527
|
...this.rowIds,
|
|
1341
|
-
...await parentIds,
|
|
1528
|
+
...await parentIds.get(),
|
|
1342
1529
|
[IS_MERGE_FIELD]: this.isMerge
|
|
1343
1530
|
};
|
|
1344
|
-
})
|
|
1531
|
+
});
|
|
1345
1532
|
this.bgLogger.log([record]);
|
|
1346
1533
|
}
|
|
1347
1534
|
logFeedback(event) {
|
|
@@ -1390,31 +1577,32 @@ var SpanImpl = class _SpanImpl {
|
|
|
1390
1577
|
return this.end(args);
|
|
1391
1578
|
}
|
|
1392
1579
|
};
|
|
1393
|
-
var Dataset = class {
|
|
1580
|
+
var Dataset = class extends ObjectFetcher {
|
|
1394
1581
|
constructor(lazyMetadata, pinnedVersion) {
|
|
1395
|
-
|
|
1582
|
+
super("dataset", pinnedVersion);
|
|
1396
1583
|
this.lazyMetadata = lazyMetadata;
|
|
1397
|
-
|
|
1398
|
-
|
|
1584
|
+
const logConn = new LazyValue(
|
|
1585
|
+
() => this.getState().then((state) => state.logConn())
|
|
1586
|
+
);
|
|
1399
1587
|
this.bgLogger = new BackgroundLogger(logConn);
|
|
1400
1588
|
}
|
|
1401
1589
|
get id() {
|
|
1402
1590
|
return (async () => {
|
|
1403
|
-
return (await this.lazyMetadata).dataset.id;
|
|
1591
|
+
return (await this.lazyMetadata.get()).dataset.id;
|
|
1404
1592
|
})();
|
|
1405
1593
|
}
|
|
1406
1594
|
get name() {
|
|
1407
1595
|
return (async () => {
|
|
1408
|
-
return (await this.lazyMetadata).dataset.name;
|
|
1596
|
+
return (await this.lazyMetadata.get()).dataset.name;
|
|
1409
1597
|
})();
|
|
1410
1598
|
}
|
|
1411
1599
|
get project() {
|
|
1412
1600
|
return (async () => {
|
|
1413
|
-
return (await this.lazyMetadata).project;
|
|
1601
|
+
return (await this.lazyMetadata.get()).project;
|
|
1414
1602
|
})();
|
|
1415
1603
|
}
|
|
1416
1604
|
async getState() {
|
|
1417
|
-
await this.lazyMetadata;
|
|
1605
|
+
await this.lazyMetadata.get();
|
|
1418
1606
|
return _state;
|
|
1419
1607
|
}
|
|
1420
1608
|
/**
|
|
@@ -1445,7 +1633,7 @@ var Dataset = class {
|
|
|
1445
1633
|
}
|
|
1446
1634
|
}
|
|
1447
1635
|
const rowId = id || v4_default();
|
|
1448
|
-
const args = (async () => ({
|
|
1636
|
+
const args = new LazyValue(async () => ({
|
|
1449
1637
|
id: rowId,
|
|
1450
1638
|
inputs: input,
|
|
1451
1639
|
output,
|
|
@@ -1453,18 +1641,18 @@ var Dataset = class {
|
|
|
1453
1641
|
dataset_id: await this.id,
|
|
1454
1642
|
created: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1455
1643
|
metadata
|
|
1456
|
-
}))
|
|
1644
|
+
}));
|
|
1457
1645
|
this.bgLogger.log([args]);
|
|
1458
1646
|
return rowId;
|
|
1459
1647
|
}
|
|
1460
1648
|
delete(id) {
|
|
1461
|
-
const args = (async () => ({
|
|
1649
|
+
const args = new LazyValue(async () => ({
|
|
1462
1650
|
id,
|
|
1463
1651
|
project_id: (await this.project).id,
|
|
1464
1652
|
dataset_id: await this.id,
|
|
1465
1653
|
created: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1466
1654
|
_object_delete: true
|
|
1467
|
-
}))
|
|
1655
|
+
}));
|
|
1468
1656
|
this.bgLogger.log([args]);
|
|
1469
1657
|
return id;
|
|
1470
1658
|
}
|
|
@@ -1500,81 +1688,6 @@ var Dataset = class {
|
|
|
1500
1688
|
dataSummary
|
|
1501
1689
|
};
|
|
1502
1690
|
}
|
|
1503
|
-
/**
|
|
1504
|
-
* Fetch all records in the dataset.
|
|
1505
|
-
*
|
|
1506
|
-
* @example
|
|
1507
|
-
* ```
|
|
1508
|
-
* // Use an async iterator to fetch all records in the dataset.
|
|
1509
|
-
* for await (const record of dataset.fetch()) {
|
|
1510
|
-
* console.log(record);
|
|
1511
|
-
* }
|
|
1512
|
-
*
|
|
1513
|
-
* // You can also iterate over the dataset directly.
|
|
1514
|
-
* for await (const record of dataset) {
|
|
1515
|
-
* console.log(record);
|
|
1516
|
-
* }
|
|
1517
|
-
* ```
|
|
1518
|
-
*
|
|
1519
|
-
* @returns An iterator over the dataset's records.
|
|
1520
|
-
*/
|
|
1521
|
-
async *fetch() {
|
|
1522
|
-
const records = await this.fetchedData();
|
|
1523
|
-
for (const record of records) {
|
|
1524
|
-
yield {
|
|
1525
|
-
id: record.id,
|
|
1526
|
-
input: record.input && JSON.parse(record.input),
|
|
1527
|
-
output: record.input && JSON.parse(record.output),
|
|
1528
|
-
metadata: record.metadata && JSON.parse(record.metadata)
|
|
1529
|
-
};
|
|
1530
|
-
}
|
|
1531
|
-
this.clearCache();
|
|
1532
|
-
}
|
|
1533
|
-
/**
|
|
1534
|
-
* Fetch all records in the dataset.
|
|
1535
|
-
*
|
|
1536
|
-
* @example
|
|
1537
|
-
* ```
|
|
1538
|
-
* // Use an async iterator to fetch all records in the dataset.
|
|
1539
|
-
* for await (const record of dataset) {
|
|
1540
|
-
* console.log(record);
|
|
1541
|
-
* }
|
|
1542
|
-
* ```
|
|
1543
|
-
*/
|
|
1544
|
-
[Symbol.asyncIterator]() {
|
|
1545
|
-
return this.fetch();
|
|
1546
|
-
}
|
|
1547
|
-
async fetchedData() {
|
|
1548
|
-
if (this._fetchedData === void 0) {
|
|
1549
|
-
const state = await this.getState();
|
|
1550
|
-
const resp = await state.logConn().get("object/dataset", {
|
|
1551
|
-
id: await this.id,
|
|
1552
|
-
fmt: "json",
|
|
1553
|
-
version: this.pinnedVersion
|
|
1554
|
-
});
|
|
1555
|
-
const text = await resp.text();
|
|
1556
|
-
this._fetchedData = text.split("\n").filter((x) => x.trim() !== "").map((x) => JSON.parse(x));
|
|
1557
|
-
}
|
|
1558
|
-
return this._fetchedData || [];
|
|
1559
|
-
}
|
|
1560
|
-
clearCache() {
|
|
1561
|
-
this._fetchedData = void 0;
|
|
1562
|
-
}
|
|
1563
|
-
async version() {
|
|
1564
|
-
if (this.pinnedVersion !== void 0) {
|
|
1565
|
-
return this.pinnedVersion;
|
|
1566
|
-
} else {
|
|
1567
|
-
const fetchedData = await this.fetchedData();
|
|
1568
|
-
let maxVersion = void 0;
|
|
1569
|
-
for (const record of fetchedData) {
|
|
1570
|
-
const xactId = record[TRANSACTION_ID_FIELD];
|
|
1571
|
-
if (maxVersion === void 0 || (xactId ?? xactId > maxVersion)) {
|
|
1572
|
-
maxVersion = xactId;
|
|
1573
|
-
}
|
|
1574
|
-
}
|
|
1575
|
-
return maxVersion;
|
|
1576
|
-
}
|
|
1577
|
-
}
|
|
1578
1691
|
/**
|
|
1579
1692
|
* Flush any pending rows to the server.
|
|
1580
1693
|
*/
|
|
@@ -1845,6 +1958,7 @@ export {
|
|
|
1845
1958
|
Logger,
|
|
1846
1959
|
NOOP_SPAN,
|
|
1847
1960
|
NoopSpan,
|
|
1961
|
+
ReadonlyExperiment,
|
|
1848
1962
|
SpanImpl,
|
|
1849
1963
|
_internalGetGlobalState,
|
|
1850
1964
|
_internalSetInitialState,
|