@langchain/classic 1.0.39 → 1.0.41

package/dist/smith/name_generation.js.map

@@ -1 +1 @@
-{"version":3,"file":"name_generation.js","names":[],"sources":["../../src/smith/name_generation.ts"],"sourcesContent":["const adjectives: string[] = [\n  \"abandoned\",\n  \"aching\",\n  \"advanced\",\n  \"ample\",\n  \"artistic\",\n  \"back\",\n  \"best\",\n  \"bold\",\n  \"brief\",\n  \"clear\",\n  \"cold\",\n  \"complicated\",\n  \"cooked\",\n  \"crazy\",\n  \"crushing\",\n  \"damp\",\n  \"dear\",\n  \"definite\",\n  \"dependable\",\n  \"diligent\",\n  \"drab\",\n  \"earnest\",\n  \"elderly\",\n  \"enchanted\",\n  \"essential\",\n  \"excellent\",\n  \"extraneous\",\n  \"fixed\",\n  \"flowery\",\n  \"formal\",\n  \"fresh\",\n  \"frosty\",\n  \"giving\",\n  \"glossy\",\n  \"healthy\",\n  \"helpful\",\n  \"impressionable\",\n  \"kind\",\n  \"large\",\n  \"left\",\n  \"long\",\n  \"loyal\",\n  \"mealy\",\n  \"memorable\",\n  \"monthly\",\n  \"new\",\n  \"notable\",\n  \"only\",\n  \"ordinary\",\n  \"passionate\",\n  \"perfect\",\n  \"pertinent\",\n  \"proper\",\n  \"puzzled\",\n  \"reflecting\",\n  \"respectful\",\n  \"roasted\",\n  \"scholarly\",\n  \"shiny\",\n  \"slight\",\n  \"sparkling\",\n  \"spotless\",\n  \"stupendous\",\n  \"sunny\",\n  \"tart\",\n  \"terrific\",\n  \"timely\",\n  \"unique\",\n  \"upbeat\",\n  \"vacant\",\n  \"virtual\",\n  \"warm\",\n  \"weary\",\n  \"whispered\",\n  \"worthwhile\",\n  \"yellow\",\n];\n\nconst nouns: string[] = [\n  \"account\",\n  \"acknowledgment\",\n  \"address\",\n  \"advertising\",\n  \"airplane\",\n  \"animal\",\n  \"appointment\",\n  \"arrival\",\n  \"artist\",\n  \"attachment\",\n  \"attitude\",\n  \"availability\",\n  \"backpack\",\n  \"bag\",\n  \"balance\",\n  \"bass\",\n  \"bean\",\n  \"beauty\",\n  \"bibliography\",\n  \"bill\",\n  \"bite\",\n  \"blossom\",\n  \"boat\",\n  \"book\",\n  \"box\",\n  \"boy\",\n  \"bread\",\n  \"bridge\",\n  \"broccoli\",\n  \"building\",\n  \"butter\",\n  \"button\",\n  \"cabbage\",\n  \"cake\",\n  \"camera\",\n  \"camp\",\n  \"candle\",\n  \"candy\",\n  \"canvas\",\n  \"car\",\n  \"card\",\n  \"carrot\",\n  \"cart\",\n  \"case\",\n  \"cat\",\n  \"chain\",\n  \"chair\",\n  \"chalk\",\n  \"chance\",\n  \"change\",\n  \"channel\",\n  \"character\",\n  \"charge\",\n  \"charm\",\n  \"chart\",\n  \"check\",\n  \"cheek\",\n  \"cheese\",\n  \"chef\",\n  \"cherry\",\n  \"chicken\",\n  \"child\",\n  \"church\",\n  \"circle\",\n  \"class\",\n  \"clay\",\n  \"click\",\n  \"clock\",\n  \"cloth\",\n  \"cloud\",\n  \"clove\",\n  \"club\",\n  \"coach\",\n  \"coal\",\n  \"coast\",\n  \"coat\",\n  \"cod\",\n  \"coffee\",\n  \"collar\",\n  \"color\",\n  \"comb\",\n  \"comfort\",\n  \"comic\",\n  \"committee\",\n  \"community\",\n  \"company\",\n  \"comparison\",\n  \"competition\",\n  \"condition\",\n  \"connection\",\n  \"control\",\n  \"cook\",\n  \"copper\",\n  \"copy\",\n  \"corn\",\n  \"cough\",\n  \"country\",\n  \"cover\",\n  \"crate\",\n  \"crayon\",\n  \"cream\",\n  \"creator\",\n  \"crew\",\n  \"crown\",\n  \"current\",\n  \"curtain\",\n  \"curve\",\n  \"cushion\",\n  \"dad\",\n  \"daughter\",\n  \"day\",\n  \"death\",\n  \"debt\",\n  \"decision\",\n  \"deer\",\n  \"degree\",\n  \"design\",\n  \"desire\",\n  \"desk\",\n  \"detail\",\n  \"development\",\n  \"digestion\",\n  \"dime\",\n  \"dinner\",\n  \"direction\",\n  \"dirt\",\n  \"discovery\",\n  \"discussion\",\n  \"disease\",\n  \"disgust\",\n  \"distance\",\n  \"distribution\",\n  \"division\",\n  \"doctor\",\n  \"dog\",\n  \"door\",\n  \"drain\",\n  \"drawer\",\n  \"dress\",\n  \"drink\",\n  \"driving\",\n  \"dust\",\n  \"ear\",\n  \"earth\",\n  \"edge\",\n  \"education\",\n  \"effect\",\n  \"egg\",\n  \"end\",\n  \"energy\",\n  \"engine\",\n  \"error\",\n  \"event\",\n  \"example\",\n  \"exchange\",\n  \"existence\",\n  \"expansion\",\n  \"experience\",\n  \"expert\",\n  \"eye\",\n  \"face\",\n  \"fact\",\n  \"fall\",\n  \"family\",\n  \"farm\",\n  \"father\",\n  \"fear\",\n  \"feeling\",\n  \"field\",\n  \"finger\",\n  \"fire\",\n  \"fish\",\n  \"flag\",\n  \"flight\",\n  \"floor\",\n  \"flower\",\n  \"fold\",\n  \"food\",\n  \"football\",\n  \"force\",\n  \"form\",\n  \"frame\",\n  \"friend\",\n  \"frog\",\n  \"fruit\",\n  \"fuel\",\n  \"furniture\",\n  \"game\",\n  \"garden\",\n  \"gate\",\n  \"girl\",\n  \"glass\",\n  \"glove\",\n  \"goat\",\n  \"gold\",\n  \"government\",\n  \"grade\",\n  \"grain\",\n  \"grass\",\n  \"green\",\n  \"grip\",\n  \"group\",\n  \"growth\",\n  \"guide\",\n  \"guitar\",\n  \"hair\",\n  \"hall\",\n  \"hand\",\n  \"harbor\",\n  \"harmony\",\n  \"hat\",\n  \"head\",\n  \"health\",\n  \"heart\",\n  \"heat\",\n  \"hill\",\n  \"history\",\n  \"hobbies\",\n  \"hole\",\n  \"hope\",\n  \"horn\",\n  \"horse\",\n  \"hospital\",\n  \"hour\",\n  \"house\",\n  \"humor\",\n  \"idea\",\n  \"impulse\",\n  \"income\",\n  \"increase\",\n  \"industry\",\n  \"ink\",\n  \"insect\",\n  \"instrument\",\n  \"insurance\",\n  \"interest\",\n  \"invention\",\n  \"iron\",\n  \"island\",\n  \"jelly\",\n  \"jet\",\n  \"jewel\",\n  \"join\",\n  \"judge\",\n  \"juice\",\n  \"jump\",\n  \"kettle\",\n  \"key\",\n  \"kick\",\n  \"kiss\",\n  \"kitten\",\n  \"knee\",\n  \"knife\",\n  \"knowledge\",\n  \"land\",\n  \"language\",\n  \"laugh\",\n  \"law\",\n  \"lead\",\n  \"learning\",\n  \"leather\",\n  \"leg\",\n  \"lettuce\",\n  \"level\",\n  \"library\",\n  \"lift\",\n  \"light\",\n  \"limit\",\n  \"line\",\n  \"linen\",\n  \"lip\",\n  \"liquid\",\n  \"list\",\n  \"look\",\n  \"loss\",\n  \"love\",\n  \"lunch\",\n  \"machine\",\n  \"man\",\n  \"manager\",\n  \"map\",\n  \"marble\",\n  \"mark\",\n  \"market\",\n  \"mass\",\n  \"match\",\n  \"meal\",\n  \"measure\",\n  \"meat\",\n  \"meeting\",\n  \"memory\",\n  \"metal\",\n  \"middle\",\n  \"milk\",\n  \"mind\",\n  \"mine\",\n  \"minute\",\n  \"mist\",\n  \"mitten\",\n  \"mom\",\n  \"money\",\n  \"monkey\",\n  \"month\",\n  \"moon\",\n  \"morning\",\n  \"mother\",\n  \"motion\",\n  \"mountain\",\n  \"mouth\",\n  \"muscle\",\n  \"music\",\n  \"nail\",\n  \"name\",\n  \"nation\",\n  \"neck\",\n  \"need\",\n  \"news\",\n  \"night\",\n  \"noise\",\n  \"note\",\n  \"number\",\n  \"nut\",\n  \"observation\",\n  \"offer\",\n  \"oil\",\n  \"operation\",\n  \"opinion\",\n  \"orange\",\n  \"order\",\n  \"organization\",\n  \"ornament\",\n  \"oven\",\n  \"page\",\n  \"pail\",\n  \"pain\",\n  \"paint\",\n  \"pan\",\n  \"pancake\",\n  \"paper\",\n  \"parcel\",\n  \"parent\",\n  \"part\",\n  \"passenger\",\n  \"paste\",\n  \"payment\",\n  \"peace\",\n  \"pear\",\n  \"pen\",\n  \"pencil\",\n  \"person\",\n  \"pest\",\n  \"pet\",\n  \"picture\",\n  \"pie\",\n  \"pin\",\n  \"pipe\",\n  \"pizza\",\n  \"place\",\n  \"plane\",\n  \"plant\",\n  \"plastic\",\n  \"plate\",\n  \"play\",\n  \"pleasure\",\n  \"plot\",\n  \"plough\",\n  \"pocket\",\n  \"point\",\n  \"poison\",\n  \"police\",\n  \"pollution\",\n  \"popcorn\",\n  \"porter\",\n  \"position\",\n  \"pot\",\n  \"potato\",\n  \"powder\",\n  \"power\",\n  \"price\",\n  \"print\",\n  \"process\",\n  \"produce\",\n  \"product\",\n  \"profit\",\n  \"property\",\n  \"prose\",\n  \"protest\",\n  \"pull\",\n  \"pump\",\n  \"punishment\",\n  \"purpose\",\n  \"push\",\n  \"quarter\",\n  \"question\",\n  \"quiet\",\n  \"quill\",\n  \"quilt\",\n  \"quince\",\n  \"rabbit\",\n  \"rail\",\n  \"rain\",\n  \"range\",\n  \"rat\",\n  \"rate\",\n  \"ray\",\n  \"reaction\",\n  \"reading\",\n  \"reason\",\n  \"record\",\n  \"regret\",\n  \"relation\",\n  \"religion\",\n  \"representative\",\n  \"request\",\n  \"respect\",\n  \"rest\",\n  \"reward\",\n  \"rhythm\",\n  \"rice\",\n  \"river\",\n  \"road\",\n  \"roll\",\n  \"room\",\n  \"root\",\n  \"rose\",\n  \"route\",\n  \"rub\",\n  \"rule\",\n  \"run\",\n  \"sack\",\n  \"sail\",\n  \"salt\",\n  \"sand\",\n  \"scale\",\n  \"scarecrow\",\n  \"scarf\",\n  \"scene\",\n  \"scent\",\n  \"school\",\n  \"science\",\n  \"scissors\",\n  \"screw\",\n  \"sea\",\n  \"seat\",\n  \"secretary\",\n  \"seed\",\n  \"selection\",\n  \"self\",\n  \"sense\",\n  \"servant\",\n  \"shade\",\n  \"shake\",\n  \"shame\",\n  \"shape\",\n  \"sheep\",\n  \"sheet\",\n  \"shelf\",\n  \"ship\",\n  \"shirt\",\n  \"shock\",\n  \"shoe\",\n  \"shop\",\n  \"show\",\n  \"side\",\n  \"sign\",\n  \"silk\",\n  \"sink\",\n  \"sister\",\n  \"size\",\n  \"sky\",\n  \"sleep\",\n  \"smash\",\n  \"smell\",\n  \"smile\",\n  \"smoke\",\n  \"snail\",\n  \"snake\",\n  \"sneeze\",\n  \"snow\",\n  \"soap\",\n  \"society\",\n  \"sock\",\n  \"soda\",\n  \"sofa\",\n  \"son\",\n  \"song\",\n  \"sort\",\n  \"sound\",\n  \"soup\",\n  \"space\",\n  \"spark\",\n  \"speed\",\n  \"sponge\",\n  \"spoon\",\n  \"spray\",\n  \"spring\",\n  \"spy\",\n  \"square\",\n  \"stamp\",\n  \"star\",\n  \"start\",\n  \"statement\",\n  \"station\",\n  \"steam\",\n  \"steel\",\n  \"stem\",\n  \"step\",\n  \"stew\",\n  \"stick\",\n  \"stitch\",\n  \"stocking\",\n  \"stomach\",\n  \"stone\",\n  \"stop\",\n  \"store\",\n  \"story\",\n  \"stove\",\n  \"stranger\",\n  \"straw\",\n  \"stream\",\n  \"street\",\n  \"stretch\",\n  \"string\",\n  \"structure\",\n  \"substance\",\n  \"sugar\",\n  \"suggestion\",\n  \"suit\",\n  \"summer\",\n  \"sun\",\n  \"support\",\n  \"surprise\",\n  \"sweater\",\n  \"swim\",\n  \"system\",\n  \"table\",\n  \"tail\",\n  \"talk\",\n  \"tank\",\n  \"taste\",\n  \"tax\",\n  \"tea\",\n  \"teaching\",\n  \"team\",\n  \"tendency\",\n  \"test\",\n  \"texture\",\n  \"theory\",\n  \"thing\",\n  \"thought\",\n  \"thread\",\n  \"throat\",\n  \"thumb\",\n  \"thunder\",\n  \"ticket\",\n  \"time\",\n  \"tin\",\n  \"title\",\n  \"toad\",\n  \"toe\",\n  \"tooth\",\n  \"toothpaste\",\n  \"touch\",\n  \"town\",\n  \"toy\",\n  \"trade\",\n  \"train\",\n  \"transport\",\n  \"tray\",\n  \"treatment\",\n  \"tree\",\n  \"trick\",\n  \"trip\",\n  \"trouble\",\n  \"trousers\",\n  \"truck\",\n  \"tub\",\n  \"turkey\",\n  \"turn\",\n  \"twist\",\n  \"umbrella\",\n  \"uncle\",\n  \"underwear\",\n  \"unit\",\n  \"use\",\n  \"vacation\",\n  \"value\",\n  \"van\",\n  \"vase\",\n  \"vegetable\",\n  \"veil\",\n  \"vein\",\n  \"verse\",\n  \"vessel\",\n  \"view\",\n  \"visitor\",\n  \"voice\",\n  \"volcano\",\n  \"walk\",\n  \"wall\",\n  \"war\",\n  \"wash\",\n  \"waste\",\n  \"watch\",\n  \"water\",\n  \"wave\",\n  \"wax\",\n  \"way\",\n  \"wealth\",\n  \"weather\",\n  \"week\",\n  \"weight\",\n  \"wheel\",\n  \"whip\",\n  \"whistle\",\n  \"window\",\n  \"wine\",\n  \"wing\",\n  \"winter\",\n  \"wire\",\n  \"wish\",\n  \"woman\",\n  \"wood\",\n  \"wool\",\n  \"word\",\n  \"work\",\n  \"worm\",\n  \"wound\",\n  \"wrist\",\n  \"writer\",\n  \"yard\",\n  \"yoke\",\n  \"zebra\",\n  \"zinc\",\n  \"zipper\",\n  \"zone\",\n];\n\nexport function randomName(): string {\n  const adjective: string =\n    adjectives[Math.floor(Math.random() * adjectives.length)];\n  const noun: string = nouns[Math.floor(Math.random() * nouns.length)];\n  const number: number = Math.floor(Math.random() * 100) + 1;\n  return `${adjective}-${noun}-${number}`;\n}\n"],"mappings":";AAAA,MAAM,aAAuB;CAC3B;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACD;AAED,MAAM,QAAkB;CACtB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACD;AAED,SAAgB,aAAqB;AAKnC,QAAO,GAHL,WAAW,KAAK,MAAM,KAAK,QAAQ,GAAG,WAAW,OAAO,EAGtC,GAFC,MAAM,KAAK,MAAM,KAAK,QAAQ,GAAG,MAAM,OAAO,EAEvC,GADL,KAAK,MAAM,KAAK,QAAQ,GAAG,IAAI,GAAG"}
+{"version":3,"file":"name_generation.js","names":[],"sources":["../../src/smith/name_generation.ts"],"sourcesContent":["const adjectives: string[] = [\n  \"abandoned\",\n  \"aching\",\n  \"advanced\",\n  \"ample\",\n  \"artistic\",\n  \"back\",\n  \"best\",\n  \"bold\",\n  \"brief\",\n  \"clear\",\n  \"cold\",\n  \"complicated\",\n  \"cooked\",\n  \"crazy\",\n  \"crushing\",\n  \"damp\",\n  \"dear\",\n  \"definite\",\n  \"dependable\",\n  \"diligent\",\n  \"drab\",\n  \"earnest\",\n  \"elderly\",\n  \"enchanted\",\n  \"essential\",\n  \"excellent\",\n  \"extraneous\",\n  \"fixed\",\n  \"flowery\",\n  \"formal\",\n  \"fresh\",\n  \"frosty\",\n  \"giving\",\n  \"glossy\",\n  \"healthy\",\n  \"helpful\",\n  \"impressionable\",\n  \"kind\",\n  \"large\",\n  \"left\",\n  \"long\",\n  \"loyal\",\n  \"mealy\",\n  \"memorable\",\n  \"monthly\",\n  \"new\",\n  \"notable\",\n  \"only\",\n  \"ordinary\",\n  \"passionate\",\n  \"perfect\",\n  \"pertinent\",\n  \"proper\",\n  \"puzzled\",\n  \"reflecting\",\n  \"respectful\",\n  \"roasted\",\n  \"scholarly\",\n  \"shiny\",\n  \"slight\",\n  \"sparkling\",\n  \"spotless\",\n  \"stupendous\",\n  \"sunny\",\n  \"tart\",\n  \"terrific\",\n  \"timely\",\n  \"unique\",\n  \"upbeat\",\n  \"vacant\",\n  \"virtual\",\n  \"warm\",\n  \"weary\",\n  \"whispered\",\n  \"worthwhile\",\n  \"yellow\",\n];\n\nconst nouns: string[] = [\n  \"account\",\n  \"acknowledgment\",\n  \"address\",\n  \"advertising\",\n  \"airplane\",\n  \"animal\",\n  \"appointment\",\n  \"arrival\",\n  \"artist\",\n  \"attachment\",\n  \"attitude\",\n  \"availability\",\n  \"backpack\",\n  \"bag\",\n  \"balance\",\n  \"bass\",\n  \"bean\",\n  \"beauty\",\n  \"bibliography\",\n  \"bill\",\n  \"bite\",\n  \"blossom\",\n  \"boat\",\n  \"book\",\n  \"box\",\n  \"boy\",\n  \"bread\",\n  \"bridge\",\n  \"broccoli\",\n  \"building\",\n  \"butter\",\n  \"button\",\n  \"cabbage\",\n  \"cake\",\n  \"camera\",\n  \"camp\",\n  \"candle\",\n  \"candy\",\n  \"canvas\",\n  \"car\",\n  \"card\",\n  \"carrot\",\n  \"cart\",\n  \"case\",\n  \"cat\",\n  \"chain\",\n  \"chair\",\n  \"chalk\",\n  \"chance\",\n  \"change\",\n  \"channel\",\n  \"character\",\n  \"charge\",\n  \"charm\",\n  \"chart\",\n  \"check\",\n  \"cheek\",\n  \"cheese\",\n  \"chef\",\n  \"cherry\",\n  \"chicken\",\n  \"child\",\n  \"church\",\n  \"circle\",\n  \"class\",\n  \"clay\",\n  \"click\",\n  \"clock\",\n  \"cloth\",\n  \"cloud\",\n  \"clove\",\n  \"club\",\n  \"coach\",\n  \"coal\",\n  \"coast\",\n  \"coat\",\n  \"cod\",\n  \"coffee\",\n  \"collar\",\n  \"color\",\n  \"comb\",\n  \"comfort\",\n  \"comic\",\n  \"committee\",\n  \"community\",\n  \"company\",\n  \"comparison\",\n  \"competition\",\n  \"condition\",\n  \"connection\",\n  \"control\",\n  \"cook\",\n  \"copper\",\n  \"copy\",\n  \"corn\",\n  \"cough\",\n  \"country\",\n  \"cover\",\n  \"crate\",\n  \"crayon\",\n  \"cream\",\n  \"creator\",\n  \"crew\",\n  \"crown\",\n  \"current\",\n  \"curtain\",\n  \"curve\",\n  \"cushion\",\n  \"dad\",\n  \"daughter\",\n  \"day\",\n  \"death\",\n  \"debt\",\n  \"decision\",\n  \"deer\",\n  \"degree\",\n  \"design\",\n  \"desire\",\n  \"desk\",\n  \"detail\",\n  \"development\",\n  \"digestion\",\n  \"dime\",\n  \"dinner\",\n  \"direction\",\n  \"dirt\",\n  \"discovery\",\n  \"discussion\",\n  \"disease\",\n  \"disgust\",\n  \"distance\",\n  \"distribution\",\n  \"division\",\n  \"doctor\",\n  \"dog\",\n  \"door\",\n  \"drain\",\n  \"drawer\",\n  \"dress\",\n  \"drink\",\n  \"driving\",\n  \"dust\",\n  \"ear\",\n  \"earth\",\n  \"edge\",\n  \"education\",\n  \"effect\",\n  \"egg\",\n  \"end\",\n  \"energy\",\n  \"engine\",\n  \"error\",\n  \"event\",\n  \"example\",\n  \"exchange\",\n  \"existence\",\n  \"expansion\",\n  \"experience\",\n  \"expert\",\n  \"eye\",\n  \"face\",\n  \"fact\",\n  \"fall\",\n  \"family\",\n  \"farm\",\n  \"father\",\n  \"fear\",\n  \"feeling\",\n  \"field\",\n  \"finger\",\n  \"fire\",\n  \"fish\",\n  \"flag\",\n  \"flight\",\n  \"floor\",\n  \"flower\",\n  \"fold\",\n  \"food\",\n  \"football\",\n  \"force\",\n  \"form\",\n  \"frame\",\n  \"friend\",\n  \"frog\",\n  \"fruit\",\n  \"fuel\",\n  \"furniture\",\n  \"game\",\n  \"garden\",\n  \"gate\",\n  \"girl\",\n  \"glass\",\n  \"glove\",\n  \"goat\",\n  \"gold\",\n  \"government\",\n  \"grade\",\n  \"grain\",\n  \"grass\",\n  \"green\",\n  \"grip\",\n  \"group\",\n  \"growth\",\n  \"guide\",\n  \"guitar\",\n  \"hair\",\n  \"hall\",\n  \"hand\",\n  \"harbor\",\n  \"harmony\",\n  \"hat\",\n  \"head\",\n  \"health\",\n  \"heart\",\n  \"heat\",\n  \"hill\",\n  \"history\",\n  \"hobbies\",\n  \"hole\",\n  \"hope\",\n  \"horn\",\n  \"horse\",\n  \"hospital\",\n  \"hour\",\n  \"house\",\n  \"humor\",\n  \"idea\",\n  \"impulse\",\n  \"income\",\n  \"increase\",\n  \"industry\",\n  \"ink\",\n  \"insect\",\n  \"instrument\",\n  \"insurance\",\n  \"interest\",\n  \"invention\",\n  \"iron\",\n  \"island\",\n  \"jelly\",\n  \"jet\",\n  \"jewel\",\n  \"join\",\n  \"judge\",\n  \"juice\",\n  \"jump\",\n  \"kettle\",\n  \"key\",\n  \"kick\",\n  \"kiss\",\n  \"kitten\",\n  \"knee\",\n  \"knife\",\n  \"knowledge\",\n  \"land\",\n  \"language\",\n  \"laugh\",\n  \"law\",\n  \"lead\",\n  \"learning\",\n  \"leather\",\n  \"leg\",\n  \"lettuce\",\n  \"level\",\n  \"library\",\n  \"lift\",\n  \"light\",\n  \"limit\",\n  \"line\",\n  \"linen\",\n  \"lip\",\n  \"liquid\",\n  \"list\",\n  \"look\",\n  \"loss\",\n  \"love\",\n  \"lunch\",\n  \"machine\",\n  \"man\",\n  \"manager\",\n  \"map\",\n  \"marble\",\n  \"mark\",\n  \"market\",\n  \"mass\",\n  \"match\",\n  \"meal\",\n  \"measure\",\n  \"meat\",\n  \"meeting\",\n  \"memory\",\n  \"metal\",\n  \"middle\",\n  \"milk\",\n  \"mind\",\n  \"mine\",\n  \"minute\",\n  \"mist\",\n  \"mitten\",\n  \"mom\",\n  \"money\",\n  \"monkey\",\n  \"month\",\n  \"moon\",\n  \"morning\",\n  \"mother\",\n  \"motion\",\n  \"mountain\",\n  \"mouth\",\n  \"muscle\",\n  \"music\",\n  \"nail\",\n  \"name\",\n  \"nation\",\n  \"neck\",\n  \"need\",\n  \"news\",\n  \"night\",\n  \"noise\",\n  \"note\",\n  \"number\",\n  \"nut\",\n  \"observation\",\n  \"offer\",\n  \"oil\",\n  \"operation\",\n  \"opinion\",\n  \"orange\",\n  \"order\",\n  \"organization\",\n  \"ornament\",\n  \"oven\",\n  \"page\",\n  \"pail\",\n  \"pain\",\n  \"paint\",\n  \"pan\",\n  \"pancake\",\n  \"paper\",\n  \"parcel\",\n  \"parent\",\n  \"part\",\n  \"passenger\",\n  \"paste\",\n  \"payment\",\n  \"peace\",\n  \"pear\",\n  \"pen\",\n  \"pencil\",\n  \"person\",\n  \"pest\",\n  \"pet\",\n  \"picture\",\n  \"pie\",\n  \"pin\",\n  \"pipe\",\n  \"pizza\",\n  \"place\",\n  \"plane\",\n  \"plant\",\n  \"plastic\",\n  \"plate\",\n  \"play\",\n  \"pleasure\",\n  \"plot\",\n  \"plough\",\n  \"pocket\",\n  \"point\",\n  \"poison\",\n  \"police\",\n  \"pollution\",\n  \"popcorn\",\n  \"porter\",\n  \"position\",\n  \"pot\",\n  \"potato\",\n  \"powder\",\n  \"power\",\n  \"price\",\n  \"print\",\n  \"process\",\n  \"produce\",\n  \"product\",\n  \"profit\",\n  \"property\",\n  \"prose\",\n  \"protest\",\n  \"pull\",\n  \"pump\",\n  \"punishment\",\n  \"purpose\",\n  \"push\",\n  \"quarter\",\n  \"question\",\n  \"quiet\",\n  \"quill\",\n  \"quilt\",\n  \"quince\",\n  \"rabbit\",\n  \"rail\",\n  \"rain\",\n  \"range\",\n  \"rat\",\n  \"rate\",\n  \"ray\",\n  \"reaction\",\n  \"reading\",\n  \"reason\",\n  \"record\",\n  \"regret\",\n  \"relation\",\n  \"religion\",\n  \"representative\",\n  \"request\",\n  \"respect\",\n  \"rest\",\n  \"reward\",\n  \"rhythm\",\n  \"rice\",\n  \"river\",\n  \"road\",\n  \"roll\",\n  \"room\",\n  \"root\",\n  \"rose\",\n  \"route\",\n  \"rub\",\n  \"rule\",\n  \"run\",\n  \"sack\",\n  \"sail\",\n  \"salt\",\n  \"sand\",\n  \"scale\",\n  \"scarecrow\",\n  \"scarf\",\n  \"scene\",\n  \"scent\",\n  \"school\",\n  \"science\",\n  \"scissors\",\n  \"screw\",\n  \"sea\",\n  \"seat\",\n  \"secretary\",\n  \"seed\",\n  \"selection\",\n  \"self\",\n  \"sense\",\n  \"servant\",\n  \"shade\",\n  \"shake\",\n  \"shame\",\n  \"shape\",\n  \"sheep\",\n  \"sheet\",\n  \"shelf\",\n  \"ship\",\n  \"shirt\",\n  \"shock\",\n  \"shoe\",\n  \"shop\",\n  \"show\",\n  \"side\",\n  \"sign\",\n  \"silk\",\n  \"sink\",\n  \"sister\",\n  \"size\",\n  \"sky\",\n  \"sleep\",\n  \"smash\",\n  \"smell\",\n  \"smile\",\n  \"smoke\",\n  \"snail\",\n  \"snake\",\n  \"sneeze\",\n  \"snow\",\n  \"soap\",\n  \"society\",\n  \"sock\",\n  \"soda\",\n  \"sofa\",\n  \"son\",\n  \"song\",\n  \"sort\",\n  \"sound\",\n  \"soup\",\n  \"space\",\n  \"spark\",\n  \"speed\",\n  \"sponge\",\n  \"spoon\",\n  \"spray\",\n  \"spring\",\n  \"spy\",\n  \"square\",\n  \"stamp\",\n  \"star\",\n  \"start\",\n  \"statement\",\n  \"station\",\n  \"steam\",\n  \"steel\",\n  \"stem\",\n  \"step\",\n  \"stew\",\n  \"stick\",\n  \"stitch\",\n  \"stocking\",\n  \"stomach\",\n  \"stone\",\n  \"stop\",\n  \"store\",\n  \"story\",\n  \"stove\",\n  \"stranger\",\n  \"straw\",\n  \"stream\",\n  \"street\",\n  \"stretch\",\n  \"string\",\n  \"structure\",\n  \"substance\",\n  \"sugar\",\n  \"suggestion\",\n  \"suit\",\n  \"summer\",\n  \"sun\",\n  \"support\",\n  \"surprise\",\n  \"sweater\",\n  \"swim\",\n  \"system\",\n  \"table\",\n  \"tail\",\n  \"talk\",\n  \"tank\",\n  \"taste\",\n  \"tax\",\n  \"tea\",\n  \"teaching\",\n  \"team\",\n  \"tendency\",\n  \"test\",\n  \"texture\",\n  \"theory\",\n  \"thing\",\n  \"thought\",\n  \"thread\",\n  \"throat\",\n  \"thumb\",\n  \"thunder\",\n  \"ticket\",\n  \"time\",\n  \"tin\",\n  \"title\",\n  \"toad\",\n  \"toe\",\n  \"tooth\",\n  \"toothpaste\",\n  \"touch\",\n  \"town\",\n  \"toy\",\n  \"trade\",\n  \"train\",\n  \"transport\",\n  \"tray\",\n  \"treatment\",\n  \"tree\",\n  \"trick\",\n  \"trip\",\n  \"trouble\",\n  \"trousers\",\n  \"truck\",\n  \"tub\",\n  \"turkey\",\n  \"turn\",\n  \"twist\",\n  \"umbrella\",\n  \"uncle\",\n  \"underwear\",\n  \"unit\",\n  \"use\",\n  \"vacation\",\n  \"value\",\n  \"van\",\n  \"vase\",\n  \"vegetable\",\n  \"veil\",\n  \"vein\",\n  \"verse\",\n  \"vessel\",\n  \"view\",\n  \"visitor\",\n  \"voice\",\n  \"volcano\",\n  \"walk\",\n  \"wall\",\n  \"war\",\n  \"wash\",\n  \"waste\",\n  \"watch\",\n  \"water\",\n  \"wave\",\n  \"wax\",\n  \"way\",\n  \"wealth\",\n  \"weather\",\n  \"week\",\n  \"weight\",\n  \"wheel\",\n  \"whip\",\n  \"whistle\",\n  \"window\",\n  \"wine\",\n  \"wing\",\n  \"winter\",\n  \"wire\",\n  \"wish\",\n  \"woman\",\n  \"wood\",\n  \"wool\",\n  \"word\",\n  \"work\",\n  \"worm\",\n  \"wound\",\n  \"wrist\",\n  \"writer\",\n  \"yard\",\n  \"yoke\",\n  \"zebra\",\n  \"zinc\",\n  \"zipper\",\n  \"zone\",\n];\n\nexport function randomName(): string {\n  const adjective: string =\n    adjectives[Math.floor(Math.random() * adjectives.length)];\n  const noun: string = nouns[Math.floor(Math.random() * nouns.length)];\n  const number: number = Math.floor(Math.random() * 100) + 1;\n  return `${adjective}-${noun}-${number}`;\n}\n"],"mappings":";AAAA,MAAM,aAAuB;CAC3B;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF;AAEA,MAAM,QAAkB;CACtB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF;AAEA,SAAgB,aAAqB;CAKnC,OAAO,GAHL,WAAW,KAAK,MAAM,KAAK,OAAO,IAAI,WAAW,MAAM,GAGrC,GAFC,MAAM,KAAK,MAAM,KAAK,OAAO,IAAI,MAAM,MAAM,GAEtC,GADL,KAAK,MAAM,KAAK,OAAO,IAAI,GAAG,IAAI;AAE3D"}

package/dist/smith/progress.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"progress.cjs","names":[],"sources":["../../src/smith/progress.ts"],"sourcesContent":["export class ProgressBar {\n  total: number;\n\n  current: number;\n\n  barLength: number;\n\n  format: string;\n\n  constructor(props: { total: number; format?: string; barLength?: number }) {\n    const { total, format, barLength } = props;\n    this.total = total;\n    this.current = 0;\n    this.barLength = barLength ?? 40;\n    this.format = format || \"{bar} {percentage}% | {value}/{total}\";\n  }\n\n  initialize(): void {\n    this.update({ current: 0 });\n  }\n\n  update({\n    current,\n    formatArgs,\n  }: {\n    current: number;\n    formatArgs?: Record<string, string>;\n  }): void {\n    this.current = current;\n    const ratio = this.current / this.total;\n    const filledBarLength = Math.round(ratio * this.barLength);\n    const emptyBarLength = this.barLength - filledBarLength;\n\n    const filledBar = \"▓\".repeat(filledBarLength);\n    const emptyBar = \"░\".repeat(emptyBarLength);\n    const percentage = (ratio * 100).toFixed(2);\n\n    let formattedString = this.format\n      .replace(\"{bar}\", `${filledBar}${emptyBar}`)\n      .replace(\"{percentage}\", percentage)\n      .replace(\"{value}\", this.current.toString())\n      .replace(\"{total}\", this.total.toString());\n    if (formatArgs) {\n      for (const key in formatArgs) {\n        if (Object.prototype.hasOwnProperty.call(formatArgs, key)) {\n          formattedString = formattedString.replace(\n            `{${key}}`,\n            formatArgs[key].toString()\n          );\n        }\n      }\n    }\n\n    console.log(formattedString);\n  }\n\n  increment({\n    formatArgs,\n  }: { formatArgs?: Record<string, string> } = {}): void {\n    this.update({ current: this.current + 1, formatArgs });\n  }\n\n  complete({ formatArgs }: { formatArgs?: Record<string, string> } = {}): void {\n    this.update({ current: this.total, formatArgs });\n    console.log(\"\\nCompleted\");\n  }\n}\n"],"mappings":";AAAA,IAAa,cAAb,MAAyB;CACvB;CAEA;CAEA;CAEA;CAEA,YAAY,OAA+D;EACzE,MAAM,EAAE,OAAO,QAAQ,cAAc;AACrC,OAAK,QAAQ;AACb,OAAK,UAAU;AACf,OAAK,YAAY,aAAa;AAC9B,OAAK,SAAS,UAAU;;CAG1B,aAAmB;AACjB,OAAK,OAAO,EAAE,SAAS,GAAG,CAAC;;CAG7B,OAAO,EACL,SACA,cAIO;AACP,OAAK,UAAU;EACf,MAAM,QAAQ,KAAK,UAAU,KAAK;EAClC,MAAM,kBAAkB,KAAK,MAAM,QAAQ,KAAK,UAAU;EAC1D,MAAM,iBAAiB,KAAK,YAAY;EAExC,MAAM,YAAY,IAAI,OAAO,gBAAgB;EAC7C,MAAM,WAAW,IAAI,OAAO,eAAe;EAC3C,MAAM,cAAc,QAAQ,KAAK,QAAQ,EAAE;EAE3C,IAAI,kBAAkB,KAAK,OACxB,QAAQ,SAAS,GAAG,YAAY,WAAW,CAC3C,QAAQ,gBAAgB,WAAW,CACnC,QAAQ,WAAW,KAAK,QAAQ,UAAU,CAAC,CAC3C,QAAQ,WAAW,KAAK,MAAM,UAAU,CAAC;AAC5C,MAAI;QACG,MAAM,OAAO,WAChB,KAAI,OAAO,UAAU,eAAe,KAAK,YAAY,IAAI,CACvD,mBAAkB,gBAAgB,QAChC,IAAI,IAAI,IACR,WAAW,KAAK,UAAU,CAC3B;;AAKP,UAAQ,IAAI,gBAAgB;;CAG9B,UAAU,EACR,eAC2C,EAAE,EAAQ;AACrD,OAAK,OAAO;GAAE,SAAS,KAAK,UAAU;GAAG;GAAY,CAAC;;CAGxD,SAAS,EAAE,eAAwD,EAAE,EAAQ;AAC3E,OAAK,OAAO;GAAE,SAAS,KAAK;GAAO;GAAY,CAAC;AAChD,UAAQ,IAAI,cAAc"}
+{"version":3,"file":"progress.cjs","names":[],"sources":["../../src/smith/progress.ts"],"sourcesContent":["export class ProgressBar {\n  total: number;\n\n  current: number;\n\n  barLength: number;\n\n  format: string;\n\n  constructor(props: { total: number; format?: string; barLength?: number }) {\n    const { total, format, barLength } = props;\n    this.total = total;\n    this.current = 0;\n    this.barLength = barLength ?? 40;\n    this.format = format || \"{bar} {percentage}% | {value}/{total}\";\n  }\n\n  initialize(): void {\n    this.update({ current: 0 });\n  }\n\n  update({\n    current,\n    formatArgs,\n  }: {\n    current: number;\n    formatArgs?: Record<string, string>;\n  }): void {\n    this.current = current;\n    const ratio = this.current / this.total;\n    const filledBarLength = Math.round(ratio * this.barLength);\n    const emptyBarLength = this.barLength - filledBarLength;\n\n    const filledBar = \"▓\".repeat(filledBarLength);\n    const emptyBar = \"░\".repeat(emptyBarLength);\n    const percentage = (ratio * 100).toFixed(2);\n\n    let formattedString = this.format\n      .replace(\"{bar}\", `${filledBar}${emptyBar}`)\n      .replace(\"{percentage}\", percentage)\n      .replace(\"{value}\", this.current.toString())\n      .replace(\"{total}\", this.total.toString());\n    if (formatArgs) {\n      for (const key in formatArgs) {\n        if (Object.prototype.hasOwnProperty.call(formatArgs, key)) {\n          formattedString = formattedString.replace(\n            `{${key}}`,\n            formatArgs[key].toString()\n          );\n        }\n      }\n    }\n\n    console.log(formattedString);\n  }\n\n  increment({\n    formatArgs,\n  }: { formatArgs?: Record<string, string> } = {}): void {\n    this.update({ current: this.current + 1, formatArgs });\n  }\n\n  complete({ formatArgs }: { formatArgs?: Record<string, string> } = {}): void {\n    this.update({ current: this.total, formatArgs });\n    console.log(\"\\nCompleted\");\n  }\n}\n"],"mappings":";AAAA,IAAa,cAAb,MAAyB;CACvB;CAEA;CAEA;CAEA;CAEA,YAAY,OAA+D;EACzE,MAAM,EAAE,OAAO,QAAQ,cAAc;EACrC,KAAK,QAAQ;EACb,KAAK,UAAU;EACf,KAAK,YAAY,aAAa;EAC9B,KAAK,SAAS,UAAU;CAC1B;CAEA,aAAmB;EACjB,KAAK,OAAO,EAAE,SAAS,EAAE,CAAC;CAC5B;CAEA,OAAO,EACL,SACA,cAIO;EACP,KAAK,UAAU;EACf,MAAM,QAAQ,KAAK,UAAU,KAAK;EAClC,MAAM,kBAAkB,KAAK,MAAM,QAAQ,KAAK,SAAS;EACzD,MAAM,iBAAiB,KAAK,YAAY;EAExC,MAAM,YAAY,IAAI,OAAO,eAAe;EAC5C,MAAM,WAAW,IAAI,OAAO,cAAc;EAC1C,MAAM,cAAc,QAAQ,IAAA,CAAK,QAAQ,CAAC;EAE1C,IAAI,kBAAkB,KAAK,OACxB,QAAQ,SAAS,GAAG,YAAY,UAAU,CAAC,CAC3C,QAAQ,gBAAgB,UAAU,CAAC,CACnC,QAAQ,WAAW,KAAK,QAAQ,SAAS,CAAC,CAAC,CAC3C,QAAQ,WAAW,KAAK,MAAM,SAAS,CAAC;EAC3C,IAAI,YACG;QAAA,MAAM,OAAO,YAChB,IAAI,OAAO,UAAU,eAAe,KAAK,YAAY,GAAG,GACtD,kBAAkB,gBAAgB,QAChC,IAAI,IAAI,IACR,WAAW,IAAI,CAAC,SAAS,CAC3B;EAAA;EAKN,QAAQ,IAAI,eAAe;CAC7B;CAEA,UAAU,EACR,eAC2C,CAAC,GAAS;EACrD,KAAK,OAAO;GAAE,SAAS,KAAK,UAAU;GAAG;EAAW,CAAC;CACvD;CAEA,SAAS,EAAE,eAAwD,CAAC,GAAS;EAC3E,KAAK,OAAO;GAAE,SAAS,KAAK;GAAO;EAAW,CAAC;EAC/C,QAAQ,IAAI,aAAa;CAC3B;AACF"}

package/dist/smith/progress.js.map

@@ -1 +1 @@
-{"version":3,"file":"progress.js","names":[],"sources":["../../src/smith/progress.ts"],"sourcesContent":["export class ProgressBar {\n  total: number;\n\n  current: number;\n\n  barLength: number;\n\n  format: string;\n\n  constructor(props: { total: number; format?: string; barLength?: number }) {\n    const { total, format, barLength } = props;\n    this.total = total;\n    this.current = 0;\n    this.barLength = barLength ?? 40;\n    this.format = format || \"{bar} {percentage}% | {value}/{total}\";\n  }\n\n  initialize(): void {\n    this.update({ current: 0 });\n  }\n\n  update({\n    current,\n    formatArgs,\n  }: {\n    current: number;\n    formatArgs?: Record<string, string>;\n  }): void {\n    this.current = current;\n    const ratio = this.current / this.total;\n    const filledBarLength = Math.round(ratio * this.barLength);\n    const emptyBarLength = this.barLength - filledBarLength;\n\n    const filledBar = \"▓\".repeat(filledBarLength);\n    const emptyBar = \"░\".repeat(emptyBarLength);\n    const percentage = (ratio * 100).toFixed(2);\n\n    let formattedString = this.format\n      .replace(\"{bar}\", `${filledBar}${emptyBar}`)\n      .replace(\"{percentage}\", percentage)\n      .replace(\"{value}\", this.current.toString())\n      .replace(\"{total}\", this.total.toString());\n    if (formatArgs) {\n      for (const key in formatArgs) {\n        if (Object.prototype.hasOwnProperty.call(formatArgs, key)) {\n          formattedString = formattedString.replace(\n            `{${key}}`,\n            formatArgs[key].toString()\n          );\n        }\n      }\n    }\n\n    console.log(formattedString);\n  }\n\n  increment({\n    formatArgs,\n  }: { formatArgs?: Record<string, string> } = {}): void {\n    this.update({ current: this.current + 1, formatArgs });\n  }\n\n  complete({ formatArgs }: { formatArgs?: Record<string, string> } = {}): void {\n    this.update({ current: this.total, formatArgs });\n    console.log(\"\\nCompleted\");\n  }\n}\n"],"mappings":";AAAA,IAAa,cAAb,MAAyB;CACvB;CAEA;CAEA;CAEA;CAEA,YAAY,OAA+D;EACzE,MAAM,EAAE,OAAO,QAAQ,cAAc;AACrC,OAAK,QAAQ;AACb,OAAK,UAAU;AACf,OAAK,YAAY,aAAa;AAC9B,OAAK,SAAS,UAAU;;CAG1B,aAAmB;AACjB,OAAK,OAAO,EAAE,SAAS,GAAG,CAAC;;CAG7B,OAAO,EACL,SACA,cAIO;AACP,OAAK,UAAU;EACf,MAAM,QAAQ,KAAK,UAAU,KAAK;EAClC,MAAM,kBAAkB,KAAK,MAAM,QAAQ,KAAK,UAAU;EAC1D,MAAM,iBAAiB,KAAK,YAAY;EAExC,MAAM,YAAY,IAAI,OAAO,gBAAgB;EAC7C,MAAM,WAAW,IAAI,OAAO,eAAe;EAC3C,MAAM,cAAc,QAAQ,KAAK,QAAQ,EAAE;EAE3C,IAAI,kBAAkB,KAAK,OACxB,QAAQ,SAAS,GAAG,YAAY,WAAW,CAC3C,QAAQ,gBAAgB,WAAW,CACnC,QAAQ,WAAW,KAAK,QAAQ,UAAU,CAAC,CAC3C,QAAQ,WAAW,KAAK,MAAM,UAAU,CAAC;AAC5C,MAAI;QACG,MAAM,OAAO,WAChB,KAAI,OAAO,UAAU,eAAe,KAAK,YAAY,IAAI,CACvD,mBAAkB,gBAAgB,QAChC,IAAI,IAAI,IACR,WAAW,KAAK,UAAU,CAC3B;;AAKP,UAAQ,IAAI,gBAAgB;;CAG9B,UAAU,EACR,eAC2C,EAAE,EAAQ;AACrD,OAAK,OAAO;GAAE,SAAS,KAAK,UAAU;GAAG;GAAY,CAAC;;CAGxD,SAAS,EAAE,eAAwD,EAAE,EAAQ;AAC3E,OAAK,OAAO;GAAE,SAAS,KAAK;GAAO;GAAY,CAAC;AAChD,UAAQ,IAAI,cAAc"}
+{"version":3,"file":"progress.js","names":[],"sources":["../../src/smith/progress.ts"],"sourcesContent":["export class ProgressBar {\n  total: number;\n\n  current: number;\n\n  barLength: number;\n\n  format: string;\n\n  constructor(props: { total: number; format?: string; barLength?: number }) {\n    const { total, format, barLength } = props;\n    this.total = total;\n    this.current = 0;\n    this.barLength = barLength ?? 40;\n    this.format = format || \"{bar} {percentage}% | {value}/{total}\";\n  }\n\n  initialize(): void {\n    this.update({ current: 0 });\n  }\n\n  update({\n    current,\n    formatArgs,\n  }: {\n    current: number;\n    formatArgs?: Record<string, string>;\n  }): void {\n    this.current = current;\n    const ratio = this.current / this.total;\n    const filledBarLength = Math.round(ratio * this.barLength);\n    const emptyBarLength = this.barLength - filledBarLength;\n\n    const filledBar = \"▓\".repeat(filledBarLength);\n    const emptyBar = \"░\".repeat(emptyBarLength);\n    const percentage = (ratio * 100).toFixed(2);\n\n    let formattedString = this.format\n      .replace(\"{bar}\", `${filledBar}${emptyBar}`)\n      .replace(\"{percentage}\", percentage)\n      .replace(\"{value}\", this.current.toString())\n      .replace(\"{total}\", this.total.toString());\n    if (formatArgs) {\n      for (const key in formatArgs) {\n        if (Object.prototype.hasOwnProperty.call(formatArgs, key)) {\n          formattedString = formattedString.replace(\n            `{${key}}`,\n            formatArgs[key].toString()\n          );\n        }\n      }\n    }\n\n    console.log(formattedString);\n  }\n\n  increment({\n    formatArgs,\n  }: { formatArgs?: Record<string, string> } = {}): void {\n    this.update({ current: this.current + 1, formatArgs });\n  }\n\n  complete({ formatArgs }: { formatArgs?: Record<string, string> } = {}): void {\n    this.update({ current: this.total, formatArgs });\n    console.log(\"\\nCompleted\");\n  }\n}\n"],"mappings":";AAAA,IAAa,cAAb,MAAyB;CACvB;CAEA;CAEA;CAEA;CAEA,YAAY,OAA+D;EACzE,MAAM,EAAE,OAAO,QAAQ,cAAc;EACrC,KAAK,QAAQ;EACb,KAAK,UAAU;EACf,KAAK,YAAY,aAAa;EAC9B,KAAK,SAAS,UAAU;CAC1B;CAEA,aAAmB;EACjB,KAAK,OAAO,EAAE,SAAS,EAAE,CAAC;CAC5B;CAEA,OAAO,EACL,SACA,cAIO;EACP,KAAK,UAAU;EACf,MAAM,QAAQ,KAAK,UAAU,KAAK;EAClC,MAAM,kBAAkB,KAAK,MAAM,QAAQ,KAAK,SAAS;EACzD,MAAM,iBAAiB,KAAK,YAAY;EAExC,MAAM,YAAY,IAAI,OAAO,eAAe;EAC5C,MAAM,WAAW,IAAI,OAAO,cAAc;EAC1C,MAAM,cAAc,QAAQ,IAAA,CAAK,QAAQ,CAAC;EAE1C,IAAI,kBAAkB,KAAK,OACxB,QAAQ,SAAS,GAAG,YAAY,UAAU,CAAC,CAC3C,QAAQ,gBAAgB,UAAU,CAAC,CACnC,QAAQ,WAAW,KAAK,QAAQ,SAAS,CAAC,CAAC,CAC3C,QAAQ,WAAW,KAAK,MAAM,SAAS,CAAC;EAC3C,IAAI,YACG;QAAA,MAAM,OAAO,YAChB,IAAI,OAAO,UAAU,eAAe,KAAK,YAAY,GAAG,GACtD,kBAAkB,gBAAgB,QAChC,IAAI,IAAI,IACR,WAAW,IAAI,CAAC,SAAS,CAC3B;EAAA;EAKN,QAAQ,IAAI,eAAe;CAC7B;CAEA,UAAU,EACR,eAC2C,CAAC,GAAS;EACrD,KAAK,OAAO;GAAE,SAAS,KAAK,UAAU;GAAG;EAAW,CAAC;CACvD;CAEA,SAAS,EAAE,eAAwD,CAAC,GAAS;EAC3E,KAAK,OAAO;GAAE,SAAS,KAAK;GAAO;EAAW,CAAC;EAC/C,QAAQ,IAAI,aAAa;CAC3B;AACF"}

package/dist/smith/runner_utils.cjs

@@ -1,4 +1,3 @@
-require("../_virtual/_rolldown/runtime.cjs");
 const require_loader = require("../evaluation/loader.cjs");
 const require_config = require("./config.cjs");
 const require_name_generation = require("./name_generation.cjs");
@@ -206,6 +205,7 @@ var PreparedRunEvaluator = class PreparedRunEvaluator {
 	}
 };
 var LoadedEvalConfig = class LoadedEvalConfig {
+	evaluators;
 	constructor(evaluators) {
 		this.evaluators = evaluators;
 	}
@@ -380,13 +380,16 @@ async function runOnDataset(chainOrFactory, datasetName, options) {
 	const runs = [];
 	for (let i = 0; i < examples.length; i += 1) runs.push(await runExtractors[i].extract());
 	let evalResults = {};
-	if (evaluationConfig) evalResults = await applyEvaluators({
-		evaluation: await LoadedEvalConfig.fromRunEvalConfig(evaluationConfig),
-		runs,
-		examples,
-		client: testClient,
-		maxConcurrency: testConcurrency
-	});
+	if (evaluationConfig) {
+		const loadedEvalConfig = await LoadedEvalConfig.fromRunEvalConfig(evaluationConfig);
+		evalResults = await applyEvaluators({
+			evaluation: loadedEvalConfig,
+			runs,
+			examples,
+			client: testClient,
+			maxConcurrency: testConcurrency
+		});
+	}
 	return {
 		projectName: testProjectName,
 		results: evalResults ?? {}

package/dist/smith/runner_utils.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"runner_utils.cjs","names":["BaseTracer","RunnableLambda","LangChainTracer","RunTree","Runnable","loadEvaluator","isCustomEvaluator","isOffTheShelfEvaluator","ProgressBar","AsyncCaller","Client","randomName"],"sources":["../../src/smith/runner_utils.ts"],"sourcesContent":["import { BaseLanguageModel } from \"@langchain/core/language_models/base\";\nimport { Serialized } from \"@langchain/core/load/serializable\";\nimport { mapStoredMessagesToChatMessages } from \"@langchain/core/messages\";\nimport {\n  Runnable,\n  RunnableConfig,\n  RunnableLambda,\n  getCallbackManagerForConfig,\n} from \"@langchain/core/runnables\";\nimport { LangChainTracer } from \"@langchain/core/tracers/tracer_langchain\";\nimport { BaseTracer } from \"@langchain/core/tracers/base\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { AsyncCaller } from \"@langchain/core/utils/async_caller\";\nimport type {\n  CallbackManager,\n  CallbackManagerForChainRun,\n} from \"@langchain/core/callbacks/manager\";\nimport {\n  Client,\n  Example,\n  Feedback,\n  Run,\n  RunTree,\n  RunTreeConfig,\n} from \"langsmith\";\nimport { EvaluationResult, RunEvaluator } from \"langsmith/evaluation\";\nimport { DataType } from \"langsmith/schemas\";\nimport type { TraceableFunction } from \"langsmith/singletons/traceable\";\nimport { LLMStringEvaluator } from \"../evaluation/base.js\";\nimport { loadEvaluator } from \"../evaluation/loader.js\";\nimport { EvaluatorType } from \"../evaluation/types.js\";\nimport {\n  isOffTheShelfEvaluator,\n  type DynamicRunEvaluatorParams,\n  type EvalConfig,\n  type EvaluatorInputFormatter,\n  type RunEvalConfig,\n  type RunEvaluatorLike,\n  isCustomEvaluator,\n} from \"./config.js\";\nimport { randomName } from \"./name_generation.js\";\nimport { ProgressBar } from \"./progress.js\";\n\nexport type ChainOrFactory =\n  | Runnable\n  | (() => Runnable)\n  | AnyTraceableFunction\n  // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n  | ((obj: any) => any)\n  // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n  | ((obj: any) => Promise<any>)\n  | (() => (obj: unknown) => unknown)\n  | (() => (obj: unknown) => Promise<unknown>);\n\nclass SingleRunIdExtractor {\n  runIdPromiseResolver: (runId: string) => void;\n\n  runIdPromise: Promise<string>;\n\n  constructor() {\n    this.runIdPromise = new Promise<string>((extract) => {\n      this.runIdPromiseResolver = extract;\n    });\n  }\n\n  handleChainStart = (\n    _chain: Serialized,\n    _inputs: ChainValues,\n    runId: string\n  ) => {\n    this.runIdPromiseResolver(runId);\n  };\n\n  async extract(): Promise<string> {\n    return this.runIdPromise;\n  }\n}\n\nclass SingleRunExtractor extends BaseTracer {\n  runPromiseResolver: (run: Run) => void;\n\n  runPromise: Promise<Run>;\n\n  /** The name of the callback handler. */\n  name = \"single_run_extractor\";\n\n  constructor() {\n    super();\n    this.runPromise = new Promise<Run>((extract) => {\n      this.runPromiseResolver = extract;\n    });\n  }\n\n  async persistRun(run: Run) {\n    this.runPromiseResolver(run);\n  }\n\n  async extract(): Promise<Run> {\n    return this.runPromise;\n  }\n}\n\n/**\n * Wraps an evaluator function + implements the RunEvaluator interface.\n */\nclass DynamicRunEvaluator implements RunEvaluator {\n  evaluator: RunnableLambda<DynamicRunEvaluatorParams, EvaluationResult>;\n\n  constructor(evaluator: RunEvaluatorLike) {\n    this.evaluator = new RunnableLambda({ func: evaluator });\n  }\n\n  /**\n   * Evaluates a run with an optional example and returns the evaluation result.\n   * @param run The run to evaluate.\n   * @param example The optional example to use for evaluation.\n   * @returns A promise that extracts to the evaluation result.\n   */\n  async evaluateRun(run: Run, example?: Example): Promise<EvaluationResult> {\n    const extractor = new SingleRunIdExtractor();\n    const tracer = new LangChainTracer({ projectName: \"evaluators\" });\n    const result = await this.evaluator.invoke(\n      {\n        run,\n        example,\n        input: run.inputs,\n        prediction: run.outputs,\n        reference: example?.outputs,\n      },\n      {\n        callbacks: [extractor, tracer],\n      }\n    );\n    const runId = await extractor.extract();\n    return {\n      sourceRunId: runId,\n      ...result,\n    };\n  }\n}\n\n// oxlint-disable-next-line @typescript-eslint/no-explicit-any\nfunction isLLMStringEvaluator(evaluator: any): evaluator is LLMStringEvaluator {\n  return evaluator && typeof evaluator.evaluateStrings === \"function\";\n}\n\n// oxlint-disable-next-line @typescript-eslint/no-explicit-any\ntype AnyTraceableFunction = TraceableFunction<(...any: any[]) => any>;\n\n/**\n * Internal implementation of RunTree, which uses the\n * provided callback manager instead of the internal LangSmith client.\n *\n * The goal of this class is to ensure seamless interop when intergrated\n * with other Runnables.\n */\nclass CallbackManagerRunTree extends RunTree {\n  callbackManager: CallbackManager;\n\n  activeCallbackManager: CallbackManagerForChainRun | undefined = undefined;\n\n  constructor(config: RunTreeConfig, callbackManager: CallbackManager) {\n    super(config);\n\n    this.callbackManager = callbackManager;\n  }\n\n  createChild(config: RunTreeConfig): CallbackManagerRunTree {\n    const child = new CallbackManagerRunTree(\n      {\n        ...config,\n        parent_run: this,\n        project_name: this.project_name,\n        client: this.client,\n      },\n      this.activeCallbackManager?.getChild() ?? this.callbackManager\n    );\n    this.child_runs.push(child);\n    return child;\n  }\n\n  async postRun(): Promise<void> {\n    // how it is translated in comparison to basic RunTree?\n    this.activeCallbackManager = await this.callbackManager.handleChainStart(\n      typeof this.serialized !== \"object\" &&\n        this.serialized != null &&\n        \"lc\" in this.serialized\n        ? this.serialized\n        : {\n            id: [\"langchain\", \"smith\", \"CallbackManagerRunTree\"],\n            lc: 1,\n            type: \"not_implemented\",\n          },\n      this.inputs,\n      this.id,\n      this.run_type,\n      undefined,\n      undefined,\n      this.name\n    );\n  }\n\n  async patchRun(): Promise<void> {\n    if (this.error) {\n      await this.activeCallbackManager?.handleChainError(\n        this.error,\n        this.id,\n        this.parent_run?.id,\n        undefined,\n        undefined\n      );\n    } else {\n      await this.activeCallbackManager?.handleChainEnd(\n        this.outputs ?? {},\n        this.id,\n        this.parent_run?.id,\n        undefined,\n        undefined\n      );\n    }\n  }\n}\n\nclass RunnableTraceable<RunInput, RunOutput> extends Runnable<\n  RunInput,\n  RunOutput\n> {\n  lc_serializable = false;\n\n  lc_namespace = [\"langchain_core\", \"runnables\"];\n\n  protected func: AnyTraceableFunction;\n\n  constructor(fields: { func: AnyTraceableFunction }) {\n    super(fields);\n\n    if (!isLangsmithTraceableFunction(fields.func)) {\n      throw new Error(\n        \"RunnableTraceable requires a function that is wrapped in traceable higher-order function\"\n      );\n    }\n\n    this.func = fields.func;\n  }\n\n  async invoke(input: RunInput, options?: Partial<RunnableConfig>) {\n    const [config] = this._getOptionsList(options ?? {}, 1);\n    const callbackManager = await getCallbackManagerForConfig(config);\n\n    const partialConfig =\n      \"langsmith:traceable\" in this.func\n        ? (this.func[\"langsmith:traceable\"] as RunTreeConfig)\n        : { name: \"<lambda>\" };\n\n    if (!callbackManager) throw new Error(\"CallbackManager not found\");\n    const runTree = new CallbackManagerRunTree(\n      {\n        ...partialConfig,\n        parent_run: callbackManager?._parentRunId\n          ? new RunTree({ name: \"<parent>\", id: callbackManager?._parentRunId })\n          : undefined,\n      },\n      callbackManager\n    );\n\n    if (\n      typeof input === \"object\" &&\n      input != null &&\n      Object.keys(input).length === 1\n    ) {\n      if (\"args\" in input && Array.isArray(input)) {\n        return (await this.func(runTree, ...input)) as RunOutput;\n      }\n\n      if (\n        \"input\" in input &&\n        !(\n          typeof input === \"object\" &&\n          input != null &&\n          !Array.isArray(input) &&\n          // oxlint-disable-next-line no-instanceof/no-instanceof\n          !(input instanceof Date)\n        )\n      ) {\n        try {\n          return (await this.func(runTree, input.input)) as RunOutput;\n        } catch {\n          return (await this.func(runTree, input)) as RunOutput;\n        }\n      }\n    }\n\n    return (await this.func(runTree, input)) as RunOutput;\n  }\n}\n\n/**\n * Wraps an off-the-shelf evaluator (loaded using loadEvaluator; of EvaluatorType[T])\n * and composes with a prepareData function so the user can prepare the trace and\n * dataset data for the evaluator.\n */\nclass PreparedRunEvaluator implements RunEvaluator {\n  evaluator: LLMStringEvaluator;\n\n  formatEvaluatorInputs: EvaluatorInputFormatter;\n\n  isStringEvaluator: boolean;\n\n  evaluationName: string;\n\n  constructor(\n    evaluator: LLMStringEvaluator,\n    evaluationName: string,\n    formatEvaluatorInputs: EvaluatorInputFormatter\n  ) {\n    this.evaluator = evaluator;\n    this.isStringEvaluator = typeof evaluator?.evaluateStrings === \"function\";\n    this.evaluationName = evaluationName;\n    this.formatEvaluatorInputs = formatEvaluatorInputs;\n  }\n\n  static async fromEvalConfig(\n    config: EvalConfig | keyof EvaluatorType\n  ): Promise<PreparedRunEvaluator> {\n    const evaluatorType =\n      typeof config === \"string\" ? config : config.evaluatorType;\n    const evalConfig = typeof config === \"string\" ? ({} as EvalConfig) : config;\n    const evaluator = await loadEvaluator(evaluatorType, evalConfig);\n    const feedbackKey = evalConfig?.feedbackKey ?? evaluator?.evaluationName;\n    if (!isLLMStringEvaluator(evaluator)) {\n      throw new Error(\n        `Evaluator of type ${evaluatorType} not yet supported. ` +\n          \"Please use a string evaluator, or implement your \" +\n          \"evaluation logic as a custom evaluator.\"\n      );\n    }\n    if (!feedbackKey) {\n      throw new Error(\n        `Evaluator of type ${evaluatorType} must have an evaluationName` +\n          ` or feedbackKey. Please manually provide a feedbackKey in the EvalConfig.`\n      );\n    }\n    return new PreparedRunEvaluator(\n      evaluator as LLMStringEvaluator,\n      feedbackKey,\n      evalConfig?.formatEvaluatorInputs\n    );\n  }\n\n  /**\n   * Evaluates a run with an optional example and returns the evaluation result.\n   * @param run The run to evaluate.\n   * @param example The optional example to use for evaluation.\n   * @returns A promise that extracts to the evaluation result.\n   */\n  async evaluateRun(run: Run, example?: Example): Promise<EvaluationResult> {\n    const { prediction, input, reference } = this.formatEvaluatorInputs({\n      rawInput: run.inputs,\n      rawPrediction: run.outputs,\n      rawReferenceOutput: example?.outputs,\n      run,\n    });\n    const extractor = new SingleRunIdExtractor();\n    const tracer = new LangChainTracer({ projectName: \"evaluators\" });\n    if (this.isStringEvaluator) {\n      const evalResult = await this.evaluator.evaluateStrings(\n        {\n          prediction: prediction as string,\n          reference: reference as string,\n          input: input as string,\n        },\n        {\n          callbacks: [extractor, tracer],\n        }\n      );\n      const runId = await extractor.extract();\n      return {\n        key: this.evaluationName,\n        comment: evalResult?.reasoning,\n        sourceRunId: runId,\n        ...evalResult,\n      };\n    }\n    throw new Error(\n      \"Evaluator not yet supported. \" +\n        \"Please use a string evaluator, or implement your \" +\n        \"evaluation logic as a custom evaluator.\"\n    );\n  }\n}\n\nclass LoadedEvalConfig {\n  constructor(public evaluators: (RunEvaluator | DynamicRunEvaluator)[]) {}\n\n  static async fromRunEvalConfig(\n    config: RunEvalConfig<keyof EvaluatorType>\n  ): Promise<LoadedEvalConfig> {\n    // Custom evaluators are applied \"as-is\"\n    const customEvaluators = (\n      config?.customEvaluators ?? config.evaluators?.filter(isCustomEvaluator)\n    )?.map((evaluator) => {\n      if (typeof evaluator === \"function\") {\n        return new DynamicRunEvaluator(evaluator);\n      } else {\n        return evaluator;\n      }\n    });\n\n    const offTheShelfEvaluators = await Promise.all(\n      config?.evaluators\n        ?.filter(isOffTheShelfEvaluator)\n        ?.map(\n          async (evaluator) =>\n            await PreparedRunEvaluator.fromEvalConfig(evaluator)\n        ) ?? []\n    );\n    return new LoadedEvalConfig(\n      (customEvaluators ?? []).concat(offTheShelfEvaluators ?? [])\n    );\n  }\n}\n\nexport interface RunOnDatasetParams extends Omit<\n  RunEvalConfig,\n  \"customEvaluators\"\n> {\n  /**\n   * Name of the project for logging and tracking.\n   */\n  projectName?: string;\n\n  /**\n   * Additional metadata for the project.\n   */\n  projectMetadata?: Record<string, unknown>;\n\n  /**\n   * Client instance for LangSmith service interaction.\n   */\n  client?: Client;\n\n  /**\n   * Maximum concurrency level for dataset processing.\n   */\n  maxConcurrency?: number;\n\n  /**\n   * @deprecated Pass keys directly to the RunOnDatasetParams instead\n   */\n  evaluationConfig?: RunEvalConfig;\n}\n\n/**\n * Internals expect a constructor () -> Runnable. This function wraps/coerces\n * the provided LangChain object, custom function, or factory function into\n * a constructor of a runnable.\n * @param modelOrFactory The model or factory to create a wrapped model from.\n * @returns A function that returns the wrapped model.\n * @throws Error if the modelOrFactory is invalid.\n */\nconst createWrappedModel = async (modelOrFactory: ChainOrFactory) => {\n  if (Runnable.isRunnable(modelOrFactory)) {\n    return () => modelOrFactory;\n  }\n  if (typeof modelOrFactory === \"function\") {\n    if (isLangsmithTraceableFunction(modelOrFactory)) {\n      const wrappedModel = new RunnableTraceable({ func: modelOrFactory });\n      return () => wrappedModel;\n    }\n\n    try {\n      // If it works with no arguments, assume it's a factory\n      let res = (modelOrFactory as () => Runnable)();\n      if (\n        res &&\n        typeof (res as unknown as Promise<Runnable>).then === \"function\"\n      ) {\n        res = await res;\n      }\n      return modelOrFactory as () => Runnable;\n    } catch {\n      // Otherwise, it's a custom UDF, and we'll wrap\n      // the function in a lambda\n      const wrappedModel = new RunnableLambda({ func: modelOrFactory });\n      return () => wrappedModel;\n    }\n  }\n  throw new Error(\"Invalid modelOrFactory\");\n};\n\nconst loadExamples = async ({\n  datasetName,\n  client,\n  projectName,\n}: {\n  datasetName: string;\n  client: Client;\n  projectName: string;\n  maxConcurrency: number;\n}) => {\n  const exampleIterator = client.listExamples({ datasetName });\n  const configs: RunnableConfig[] = [];\n  const runExtractors = [];\n  const examples = [];\n  for await (const example of exampleIterator) {\n    const runExtractor = new SingleRunExtractor();\n    configs.push({\n      callbacks: [\n        new LangChainTracer({ exampleId: example.id, projectName }),\n        runExtractor,\n      ],\n    });\n    examples.push(example);\n    runExtractors.push(runExtractor);\n  }\n  return {\n    configs,\n    examples,\n    runExtractors,\n  };\n};\n\nconst applyEvaluators = async ({\n  evaluation,\n  runs,\n  examples,\n  client,\n  maxConcurrency,\n}: {\n  evaluation: LoadedEvalConfig;\n  runs: Run[];\n  examples: Example[];\n  client: Client;\n  maxConcurrency: number;\n}): Promise<{\n  [key: string]: {\n    execution_time?: number;\n    run_id: string;\n    feedback: Feedback[];\n  };\n}> => {\n  // TODO: Parallelize and/or put in callbacks to speed up evals.\n  const { evaluators } = evaluation;\n  const progress = new ProgressBar({\n    total: examples.length,\n    format: \"Running Evaluators: {bar} {percentage}% | {value}/{total}\\n\",\n  });\n  const caller = new AsyncCaller({\n    maxConcurrency,\n  });\n  const requests = runs.map(\n    async (\n      run,\n      i\n    ): Promise<{\n      run_id: string;\n      execution_time?: number;\n      feedback: Feedback[];\n    }> =>\n      caller.call(async () => {\n        const evaluatorResults = await Promise.allSettled(\n          evaluators.map((evaluator) =>\n            client.evaluateRun(run, evaluator, {\n              referenceExample: examples[i],\n              loadChildRuns: false,\n            })\n          )\n        );\n        progress.increment();\n        return {\n          execution_time:\n            run?.end_time && run.start_time\n              ? new Date(run.end_time).getTime() -\n                new Date(run.start_time).getTime()\n              : undefined,\n          feedback: evaluatorResults.map((evalResult) =>\n            evalResult.status === \"fulfilled\"\n              ? evalResult.value\n              : evalResult.reason\n          ),\n          run_id: run.id,\n        };\n      })\n  );\n  const results = await Promise.all(requests);\n\n  return results.reduce(\n    (acc, result, i) => ({\n      ...acc,\n      [examples[i].id]: result,\n    }),\n    {}\n  );\n};\n\nexport type EvalResults = {\n  projectName: string;\n  results: {\n    [key: string]: {\n      execution_time?: number;\n      run_id: string;\n      feedback: Feedback[];\n    };\n  };\n};\n\nconst getExamplesInputs = (\n  examples: Example[],\n  chainOrFactory: ChainOrFactory,\n  dataType?: DataType\n) => {\n  if (dataType === \"chat\") {\n    // For some batty reason, we store the chat dataset differently.\n    // { type: \"system\", data: { content: inputs.input } },\n    // But we need to create AIMesage, SystemMessage, etc.\n    return examples.map(({ inputs }) =>\n      mapStoredMessagesToChatMessages(inputs.input)\n    );\n  }\n  // If it's a language model and ALL example inputs have a single value,\n  // then we can be friendly and flatten the inputs to a list of strings.\n  const isLanguageModel =\n    typeof chainOrFactory === \"object\" &&\n    typeof (chainOrFactory as BaseLanguageModel)._llmType === \"function\";\n  if (\n    isLanguageModel &&\n    examples.every(({ inputs }) => Object.keys(inputs).length === 1)\n  ) {\n    return examples.map(({ inputs }) => Object.values(inputs)[0]);\n  }\n  return examples.map(({ inputs }) => inputs);\n};\n\n/**\n * Evaluates a given model or chain against a specified LangSmith dataset.\n *\n * This function fetches example records from the specified dataset,\n * runs the model or chain against each example, and returns the evaluation\n * results.\n *\n * @param chainOrFactory - A model or factory/constructor function to be evaluated. It can be a\n * Runnable instance, a factory function that returns a Runnable, or a user-defined\n * function or factory.\n *\n * @param datasetName - The name of the dataset against which the evaluation will be\n * performed. This dataset should already be defined and contain the relevant data\n * for evaluation.\n *\n * @param options - (Optional) Additional parameters for the evaluation process:\n *   - `evaluators` (RunEvalType[]): Evaluators to apply to a dataset run.\n *   - `formatEvaluatorInputs` (EvaluatorInputFormatter): Convert the evaluation data into formats that can be used by the evaluator.\n *   - `projectName` (string): Name of the project for logging and tracking.\n *   - `projectMetadata` (Record<string, unknown>): Additional metadata for the project.\n *   - `client` (Client): Client instance for LangSmith service interaction.\n *   - `maxConcurrency` (number): Maximum concurrency level for dataset processing.\n *\n * @returns A promise that resolves to an `EvalResults` object. This object includes\n * detailed results of the evaluation, such as execution time, run IDs, and feedback\n * for each entry in the dataset.\n *\n * @example\n * ```typescript\n * // Example usage for evaluating a model on a dataset\n * async function evaluateModel() {\n *   const chain = /* ...create your model or chain...*\\//\n *   const datasetName = 'example-dataset';\n *   const client = new Client(/* ...config... *\\//);\n *\n *   const results = await runOnDataset(chain, datasetName, {\n *     evaluators: [/* ...evaluators... *\\//],\n *     client,\n *   });\n *\n *   console.log('Evaluation Results:', results);\n * }\n *\n * evaluateModel();\n * ```\n * In this example, `runOnDataset` is used to evaluate a language model (or a chain of models) against\n * a dataset named 'example-dataset'. The evaluation process is configured using `RunOnDatasetParams[\"evaluators\"]`, which can\n * include both standard and custom evaluators. The `Client` instance is used to interact with LangChain services.\n * The function returns the evaluation results, which can be logged or further processed as needed.\n */\n\nexport async function runOnDataset(\n  chainOrFactory: ChainOrFactory,\n  datasetName: string,\n  options?: RunOnDatasetParams\n) {\n  const {\n    projectName,\n    projectMetadata,\n    client,\n    maxConcurrency,\n  }: RunOnDatasetParams = options ?? {};\n\n  const evaluationConfig: RunEvalConfig | undefined =\n    options?.evaluationConfig ??\n    (options?.evaluators != null\n      ? {\n          evaluators: options.evaluators,\n          formatEvaluatorInputs: options.formatEvaluatorInputs,\n        }\n      : undefined);\n\n  const wrappedModel = await createWrappedModel(chainOrFactory);\n  const testClient = client ?? new Client();\n  const testProjectName = projectName ?? randomName();\n  const dataset = await testClient.readDataset({ datasetName });\n  const datasetId = dataset.id;\n  const testConcurrency = maxConcurrency ?? 5;\n  const { configs, examples, runExtractors } = await loadExamples({\n    datasetName,\n    client: testClient,\n    projectName: testProjectName,\n    maxConcurrency: testConcurrency,\n  });\n\n  await testClient.createProject({\n    projectName: testProjectName,\n    referenceDatasetId: datasetId,\n    projectExtra: { metadata: { ...projectMetadata } },\n  });\n  const wrappedRunnable: Runnable = new RunnableLambda({\n    func: wrappedModel,\n  }).withConfig({ runName: \"evaluationRun\" });\n  const runInputs = getExamplesInputs(\n    examples,\n    chainOrFactory,\n    dataset.data_type\n  );\n  const progress = new ProgressBar({\n    total: runInputs.length,\n    format: \"Predicting: {bar} {percentage}% | {value}/{total}\",\n  });\n  // TODO: Collect the runs as well.\n  await wrappedRunnable\n    .withListeners({\n      onEnd: () => progress.increment(),\n    })\n    // TODO: Insert evaluation inline for immediate feedback.\n    .batch(runInputs, configs, {\n      maxConcurrency,\n      returnExceptions: true,\n    });\n\n  progress.complete();\n  const runs: Run[] = [];\n  for (let i = 0; i < examples.length; i += 1) {\n    runs.push(await runExtractors[i].extract());\n  }\n  let evalResults: Record<\n    string,\n    { run_id: string; execution_time?: number; feedback: Feedback[] }\n  > = {};\n  if (evaluationConfig) {\n    const loadedEvalConfig =\n      await LoadedEvalConfig.fromRunEvalConfig(evaluationConfig);\n    evalResults = await applyEvaluators({\n      evaluation: loadedEvalConfig,\n      runs,\n      examples,\n      client: testClient,\n      maxConcurrency: testConcurrency,\n    });\n  }\n  const results: EvalResults = {\n    projectName: testProjectName,\n    results: evalResults ?? {},\n  };\n  return results;\n}\n\nfunction isLangsmithTraceableFunction(x: unknown): x is AnyTraceableFunction {\n  return typeof x === \"function\" && \"langsmith:traceable\" in x;\n}\n"],"mappings":";;;;;;;;;;;;AAsDA,IAAM,uBAAN,MAA2B;CACzB;CAEA;CAEA,cAAc;AACZ,OAAK,eAAe,IAAI,SAAiB,YAAY;AACnD,QAAK,uBAAuB;IAC5B;;CAGJ,oBACE,QACA,SACA,UACG;AACH,OAAK,qBAAqB,MAAM;;CAGlC,MAAM,UAA2B;AAC/B,SAAO,KAAK;;;AAIhB,IAAM,qBAAN,cAAiCA,6BAAAA,WAAW;CAC1C;CAEA;;CAGA,OAAO;CAEP,cAAc;AACZ,SAAO;AACP,OAAK,aAAa,IAAI,SAAc,YAAY;AAC9C,QAAK,qBAAqB;IAC1B;;CAGJ,MAAM,WAAW,KAAU;AACzB,OAAK,mBAAmB,IAAI;;CAG9B,MAAM,UAAwB;AAC5B,SAAO,KAAK;;;;;;AAOhB,IAAM,sBAAN,MAAkD;CAChD;CAEA,YAAY,WAA6B;AACvC,OAAK,YAAY,IAAIC,0BAAAA,eAAe,EAAE,MAAM,WAAW,CAAC;;;;;;;;CAS1D,MAAM,YAAY,KAAU,SAA8C;EACxE,MAAM,YAAY,IAAI,sBAAsB;EAC5C,MAAM,SAAS,IAAIC,yCAAAA,gBAAgB,EAAE,aAAa,cAAc,CAAC;EACjE,MAAM,SAAS,MAAM,KAAK,UAAU,OAClC;GACE;GACA;GACA,OAAO,IAAI;GACX,YAAY,IAAI;GAChB,WAAW,SAAS;GACrB,EACD,EACE,WAAW,CAAC,WAAW,OAAO,EAC/B,CACF;AAED,SAAO;GACL,aAAa,MAFK,UAAU,SAAS;GAGrC,GAAG;GACJ;;;AAKL,SAAS,qBAAqB,WAAiD;AAC7E,QAAO,aAAa,OAAO,UAAU,oBAAoB;;;;;;;;;AAa3D,IAAM,yBAAN,MAAM,+BAA+BC,UAAAA,QAAQ;CAC3C;CAEA,wBAAgE,KAAA;CAEhE,YAAY,QAAuB,iBAAkC;AACnE,QAAM,OAAO;AAEb,OAAK,kBAAkB;;CAGzB,YAAY,QAA+C;EACzD,MAAM,QAAQ,IAAI,uBAChB;GACE,GAAG;GACH,YAAY;GACZ,cAAc,KAAK;GACnB,QAAQ,KAAK;GACd,EACD,KAAK,uBAAuB,UAAU,IAAI,KAAK,gBAChD;AACD,OAAK,WAAW,KAAK,MAAM;AAC3B,SAAO;;CAGT,MAAM,UAAyB;AAE7B,OAAK,wBAAwB,MAAM,KAAK,gBAAgB,iBACtD,OAAO,KAAK,eAAe,YACzB,KAAK,cAAc,QACnB,QAAQ,KAAK,aACX,KAAK,aACL;GACE,IAAI;IAAC;IAAa;IAAS;IAAyB;GACpD,IAAI;GACJ,MAAM;GACP,EACL,KAAK,QACL,KAAK,IACL,KAAK,UACL,KAAA,GACA,KAAA,GACA,KAAK,KACN;;CAGH,MAAM,WAA0B;AAC9B,MAAI,KAAK,MACP,OAAM,KAAK,uBAAuB,iBAChC,KAAK,OACL,KAAK,IACL,KAAK,YAAY,IACjB,KAAA,GACA,KAAA,EACD;MAED,OAAM,KAAK,uBAAuB,eAChC,KAAK,WAAW,EAAE,EAClB,KAAK,IACL,KAAK,YAAY,IACjB,KAAA,GACA,KAAA,EACD;;;AAKP,IAAM,oBAAN,cAAqDC,0BAAAA,SAGnD;CACA,kBAAkB;CAElB,eAAe,CAAC,kBAAkB,YAAY;CAE9C;CAEA,YAAY,QAAwC;AAClD,QAAM,OAAO;AAEb,MAAI,CAAC,6BAA6B,OAAO,KAAK,CAC5C,OAAM,IAAI,MACR,2FACD;AAGH,OAAK,OAAO,OAAO;;CAGrB,MAAM,OAAO,OAAiB,SAAmC;EAC/D,MAAM,CAAC,UAAU,KAAK,gBAAgB,WAAW,EAAE,EAAE,EAAE;EACvD,MAAM,kBAAkB,OAAA,GAAA,0BAAA,6BAAkC,OAAO;EAEjE,MAAM,gBACJ,yBAAyB,KAAK,OACzB,KAAK,KAAK,yBACX,EAAE,MAAM,YAAY;AAE1B,MAAI,CAAC,gBAAiB,OAAM,IAAI,MAAM,4BAA4B;EAClE,MAAM,UAAU,IAAI,uBAClB;GACE,GAAG;GACH,YAAY,iBAAiB,eACzB,IAAID,UAAAA,QAAQ;IAAE,MAAM;IAAY,IAAI,iBAAiB;IAAc,CAAC,GACpE,KAAA;GACL,EACD,gBACD;AAED,MACE,OAAO,UAAU,YACjB,SAAS,QACT,OAAO,KAAK,MAAM,CAAC,WAAW,GAC9B;AACA,OAAI,UAAU,SAAS,MAAM,QAAQ,MAAM,CACzC,QAAQ,MAAM,KAAK,KAAK,SAAS,GAAG,MAAM;AAG5C,OACE,WAAW,SACX,EACE,OAAO,UAAU,YACjB,SAAS,QACT,CAAC,MAAM,QAAQ,MAAM,IAErB,EAAE,iBAAiB,OAGrB,KAAI;AACF,WAAQ,MAAM,KAAK,KAAK,SAAS,MAAM,MAAM;WACvC;AACN,WAAQ,MAAM,KAAK,KAAK,SAAS,MAAM;;;AAK7C,SAAQ,MAAM,KAAK,KAAK,SAAS,MAAM;;;;;;;;AAS3C,IAAM,uBAAN,MAAM,qBAA6C;CACjD;CAEA;CAEA;CAEA;CAEA,YACE,WACA,gBACA,uBACA;AACA,OAAK,YAAY;AACjB,OAAK,oBAAoB,OAAO,WAAW,oBAAoB;AAC/D,OAAK,iBAAiB;AACtB,OAAK,wBAAwB;;CAG/B,aAAa,eACX,QAC+B;EAC/B,MAAM,gBACJ,OAAO,WAAW,WAAW,SAAS,OAAO;EAC/C,MAAM,aAAa,OAAO,WAAW,WAAY,EAAE,GAAkB;EACrE,MAAM,YAAY,MAAME,eAAAA,cAAc,eAAe,WAAW;EAChE,MAAM,cAAc,YAAY,eAAe,WAAW;AAC1D,MAAI,CAAC,qBAAqB,UAAU,CAClC,OAAM,IAAI,MACR,qBAAqB,cAAc,8GAGpC;AAEH,MAAI,CAAC,YACH,OAAM,IAAI,MACR,qBAAqB,cAAc,uGAEpC;AAEH,SAAO,IAAI,qBACT,WACA,aACA,YAAY,sBACb;;;;;;;;CASH,MAAM,YAAY,KAAU,SAA8C;EACxE,MAAM,EAAE,YAAY,OAAO,cAAc,KAAK,sBAAsB;GAClE,UAAU,IAAI;GACd,eAAe,IAAI;GACnB,oBAAoB,SAAS;GAC7B;GACD,CAAC;EACF,MAAM,YAAY,IAAI,sBAAsB;EAC5C,MAAM,SAAS,IAAIH,yCAAAA,gBAAgB,EAAE,aAAa,cAAc,CAAC;AACjE,MAAI,KAAK,mBAAmB;GAC1B,MAAM,aAAa,MAAM,KAAK,UAAU,gBACtC;IACc;IACD;IACJ;IACR,EACD,EACE,WAAW,CAAC,WAAW,OAAO,EAC/B,CACF;GACD,MAAM,QAAQ,MAAM,UAAU,SAAS;AACvC,UAAO;IACL,KAAK,KAAK;IACV,SAAS,YAAY;IACrB,aAAa;IACb,GAAG;IACJ;;AAEH,QAAM,IAAI,MACR,wHAGD;;;AAIL,IAAM,mBAAN,MAAM,iBAAiB;CACrB,YAAY,YAA2D;AAApD,OAAA,aAAA;;CAEnB,aAAa,kBACX,QAC2B;EAE3B,MAAM,oBACJ,QAAQ,oBAAoB,OAAO,YAAY,OAAOI,eAAAA,kBAAkB,GACvE,KAAK,cAAc;AACpB,OAAI,OAAO,cAAc,WACvB,QAAO,IAAI,oBAAoB,UAAU;OAEzC,QAAO;IAET;EAEF,MAAM,wBAAwB,MAAM,QAAQ,IAC1C,QAAQ,YACJ,OAAOC,eAAAA,uBAAuB,EAC9B,IACA,OAAO,cACL,MAAM,qBAAqB,eAAe,UAAU,CACvD,IAAI,EAAE,CACV;AACD,SAAO,IAAI,kBACR,oBAAoB,EAAE,EAAE,OAAO,yBAAyB,EAAE,CAAC,CAC7D;;;;;;;;;;;AA0CL,MAAM,qBAAqB,OAAO,mBAAmC;AACnE,KAAIH,0BAAAA,SAAS,WAAW,eAAe,CACrC,cAAa;AAEf,KAAI,OAAO,mBAAmB,YAAY;AACxC,MAAI,6BAA6B,eAAe,EAAE;GAChD,MAAM,eAAe,IAAI,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AACpE,gBAAa;;AAGf,MAAI;GAEF,IAAI,MAAO,gBAAmC;AAC9C,OACE,OACA,OAAQ,IAAqC,SAAS,WAEtD,OAAM,MAAM;AAEd,UAAO;UACD;GAGN,MAAM,eAAe,IAAIH,0BAAAA,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACjE,gBAAa;;;AAGjB,OAAM,IAAI,MAAM,yBAAyB;;AAG3C,MAAM,eAAe,OAAO,EAC1B,aACA,QACA,kBAMI;CACJ,MAAM,kBAAkB,OAAO,aAAa,EAAE,aAAa,CAAC;CAC5D,MAAM,UAA4B,EAAE;CACpC,MAAM,gBAAgB,EAAE;CACxB,MAAM,WAAW,EAAE;AACnB,YAAW,MAAM,WAAW,iBAAiB;EAC3C,MAAM,eAAe,IAAI,oBAAoB;AAC7C,UAAQ,KAAK,EACX,WAAW,CACT,IAAIC,yCAAAA,gBAAgB;GAAE,WAAW,QAAQ;GAAI;GAAa,CAAC,EAC3D,aACD,EACF,CAAC;AACF,WAAS,KAAK,QAAQ;AACtB,gBAAc,KAAK,aAAa;;AAElC,QAAO;EACL;EACA;EACA;EACD;;AAGH,MAAM,kBAAkB,OAAO,EAC7B,YACA,MACA,UACA,QACA,qBAaI;CAEJ,MAAM,EAAE,eAAe;CACvB,MAAM,WAAW,IAAIM,iBAAAA,YAAY;EAC/B,OAAO,SAAS;EAChB,QAAQ;EACT,CAAC;CACF,MAAM,SAAS,IAAIC,mCAAAA,YAAY,EAC7B,gBACD,CAAC;CACF,MAAM,WAAW,KAAK,IACpB,OACE,KACA,MAMA,OAAO,KAAK,YAAY;EACtB,MAAM,mBAAmB,MAAM,QAAQ,WACrC,WAAW,KAAK,cACd,OAAO,YAAY,KAAK,WAAW;GACjC,kBAAkB,SAAS;GAC3B,eAAe;GAChB,CAAC,CACH,CACF;AACD,WAAS,WAAW;AACpB,SAAO;GACL,gBACE,KAAK,YAAY,IAAI,aACjB,IAAI,KAAK,IAAI,SAAS,CAAC,SAAS,GAChC,IAAI,KAAK,IAAI,WAAW,CAAC,SAAS,GAClC,KAAA;GACN,UAAU,iBAAiB,KAAK,eAC9B,WAAW,WAAW,cAClB,WAAW,QACX,WAAW,OAChB;GACD,QAAQ,IAAI;GACb;GACD,CACL;AAGD,SAAO,MAFe,QAAQ,IAAI,SAAS,EAE5B,QACZ,KAAK,QAAQ,OAAO;EACnB,GAAG;GACF,SAAS,GAAG,KAAK;EACnB,GACD,EAAE,CACH;;AAcH,MAAM,qBACJ,UACA,gBACA,aACG;AACH,KAAI,aAAa,OAIf,QAAO,SAAS,KAAK,EAAE,cAAA,GAAA,yBAAA,iCACW,OAAO,MAAM,CAC9C;AAOH,KAFE,OAAO,mBAAmB,YAC1B,OAAQ,eAAqC,aAAa,cAG1D,SAAS,OAAO,EAAE,aAAa,OAAO,KAAK,OAAO,CAAC,WAAW,EAAE,CAEhE,QAAO,SAAS,KAAK,EAAE,aAAa,OAAO,OAAO,OAAO,CAAC,GAAG;AAE/D,QAAO,SAAS,KAAK,EAAE,aAAa,OAAO;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAsD7C,eAAsB,aACpB,gBACA,aACA,SACA;CACA,MAAM,EACJ,aACA,iBACA,QACA,mBACsB,WAAW,EAAE;CAErC,MAAM,mBACJ,SAAS,qBACR,SAAS,cAAc,OACpB;EACE,YAAY,QAAQ;EACpB,uBAAuB,QAAQ;EAChC,GACD,KAAA;CAEN,MAAM,eAAe,MAAM,mBAAmB,eAAe;CAC7D,MAAM,aAAa,UAAU,IAAIC,UAAAA,QAAQ;CACzC,MAAM,kBAAkB,eAAeC,wBAAAA,YAAY;CACnD,MAAM,UAAU,MAAM,WAAW,YAAY,EAAE,aAAa,CAAC;CAC7D,MAAM,YAAY,QAAQ;CAC1B,MAAM,kBAAkB,kBAAkB;CAC1C,MAAM,EAAE,SAAS,UAAU,kBAAkB,MAAM,aAAa;EAC9D;EACA,QAAQ;EACR,aAAa;EACb,gBAAgB;EACjB,CAAC;AAEF,OAAM,WAAW,cAAc;EAC7B,aAAa;EACb,oBAAoB;EACpB,cAAc,EAAE,UAAU,EAAE,GAAG,iBAAiB,EAAE;EACnD,CAAC;CACF,MAAM,kBAA4B,IAAIV,0BAAAA,eAAe,EACnD,MAAM,cACP,CAAC,CAAC,WAAW,EAAE,SAAS,iBAAiB,CAAC;CAC3C,MAAM,YAAY,kBAChB,UACA,gBACA,QAAQ,UACT;CACD,MAAM,WAAW,IAAIO,iBAAAA,YAAY;EAC/B,OAAO,UAAU;EACjB,QAAQ;EACT,CAAC;AAEF,OAAM,gBACH,cAAc,EACb,aAAa,SAAS,WAAW,EAClC,CAAC,CAED,MAAM,WAAW,SAAS;EACzB;EACA,kBAAkB;EACnB,CAAC;AAEJ,UAAS,UAAU;CACnB,MAAM,OAAc,EAAE;AACtB,MAAK,IAAI,IAAI,GAAG,IAAI,SAAS,QAAQ,KAAK,EACxC,MAAK,KAAK,MAAM,cAAc,GAAG,SAAS,CAAC;CAE7C,IAAI,cAGA,EAAE;AACN,KAAI,iBAGF,eAAc,MAAM,gBAAgB;EAClC,YAAY,MAFN,iBAAiB,kBAAkB,iBAAiB;EAG1D;EACA;EACA,QAAQ;EACR,gBAAgB;EACjB,CAAC;AAMJ,QAAO;EAHL,aAAa;EACb,SAAS,eAAe,EAAE;EAEd;;AAGhB,SAAS,6BAA6B,GAAuC;AAC3E,QAAO,OAAO,MAAM,cAAc,yBAAyB"}
+{"version":3,"file":"runner_utils.cjs","names":["BaseTracer","RunnableLambda","LangChainTracer","RunTree","Runnable","loadEvaluator","isCustomEvaluator","isOffTheShelfEvaluator","ProgressBar","AsyncCaller","Client","randomName"],"sources":["../../src/smith/runner_utils.ts"],"sourcesContent":["import { BaseLanguageModel } from \"@langchain/core/language_models/base\";\nimport { Serialized } from \"@langchain/core/load/serializable\";\nimport { mapStoredMessagesToChatMessages } from \"@langchain/core/messages\";\nimport {\n  Runnable,\n  RunnableConfig,\n  RunnableLambda,\n  getCallbackManagerForConfig,\n} from \"@langchain/core/runnables\";\nimport { LangChainTracer } from \"@langchain/core/tracers/tracer_langchain\";\nimport { BaseTracer } from \"@langchain/core/tracers/base\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { AsyncCaller } from \"@langchain/core/utils/async_caller\";\nimport type {\n  CallbackManager,\n  CallbackManagerForChainRun,\n} from \"@langchain/core/callbacks/manager\";\nimport {\n  Client,\n  Example,\n  Feedback,\n  Run,\n  RunTree,\n  RunTreeConfig,\n} from \"langsmith\";\nimport { EvaluationResult, RunEvaluator } from \"langsmith/evaluation\";\nimport { DataType } from \"langsmith/schemas\";\nimport type { TraceableFunction } from \"langsmith/singletons/traceable\";\nimport { LLMStringEvaluator } from \"../evaluation/base.js\";\nimport { loadEvaluator } from \"../evaluation/loader.js\";\nimport { EvaluatorType } from \"../evaluation/types.js\";\nimport {\n  isOffTheShelfEvaluator,\n  type DynamicRunEvaluatorParams,\n  type EvalConfig,\n  type EvaluatorInputFormatter,\n  type RunEvalConfig,\n  type RunEvaluatorLike,\n  isCustomEvaluator,\n} from \"./config.js\";\nimport { randomName } from \"./name_generation.js\";\nimport { ProgressBar } from \"./progress.js\";\n\nexport type ChainOrFactory =\n  | Runnable\n  | (() => Runnable)\n  | AnyTraceableFunction\n  // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n  | ((obj: any) => any)\n  // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n  | ((obj: any) => Promise<any>)\n  | (() => (obj: unknown) => unknown)\n  | (() => (obj: unknown) => Promise<unknown>);\n\nclass SingleRunIdExtractor {\n  runIdPromiseResolver: (runId: string) => void;\n\n  runIdPromise: Promise<string>;\n\n  constructor() {\n    this.runIdPromise = new Promise<string>((extract) => {\n      this.runIdPromiseResolver = extract;\n    });\n  }\n\n  handleChainStart = (\n    _chain: Serialized,\n    _inputs: ChainValues,\n    runId: string\n  ) => {\n    this.runIdPromiseResolver(runId);\n  };\n\n  async extract(): Promise<string> {\n    return this.runIdPromise;\n  }\n}\n\nclass SingleRunExtractor extends BaseTracer {\n  runPromiseResolver: (run: Run) => void;\n\n  runPromise: Promise<Run>;\n\n  /** The name of the callback handler. */\n  name = \"single_run_extractor\";\n\n  constructor() {\n    super();\n    this.runPromise = new Promise<Run>((extract) => {\n      this.runPromiseResolver = extract;\n    });\n  }\n\n  async persistRun(run: Run) {\n    this.runPromiseResolver(run);\n  }\n\n  async extract(): Promise<Run> {\n    return this.runPromise;\n  }\n}\n\n/**\n * Wraps an evaluator function + implements the RunEvaluator interface.\n */\nclass DynamicRunEvaluator implements RunEvaluator {\n  evaluator: RunnableLambda<DynamicRunEvaluatorParams, EvaluationResult>;\n\n  constructor(evaluator: RunEvaluatorLike) {\n    this.evaluator = new RunnableLambda({ func: evaluator });\n  }\n\n  /**\n   * Evaluates a run with an optional example and returns the evaluation result.\n   * @param run The run to evaluate.\n   * @param example The optional example to use for evaluation.\n   * @returns A promise that extracts to the evaluation result.\n   */\n  async evaluateRun(run: Run, example?: Example): Promise<EvaluationResult> {\n    const extractor = new SingleRunIdExtractor();\n    const tracer = new LangChainTracer({ projectName: \"evaluators\" });\n    const result = await this.evaluator.invoke(\n      {\n        run,\n        example,\n        input: run.inputs,\n        prediction: run.outputs,\n        reference: example?.outputs,\n      },\n      {\n        callbacks: [extractor, tracer],\n      }\n    );\n    const runId = await extractor.extract();\n    return {\n      sourceRunId: runId,\n      ...result,\n    };\n  }\n}\n\n// oxlint-disable-next-line @typescript-eslint/no-explicit-any\nfunction isLLMStringEvaluator(evaluator: any): evaluator is LLMStringEvaluator {\n  return evaluator && typeof evaluator.evaluateStrings === \"function\";\n}\n\n// oxlint-disable-next-line @typescript-eslint/no-explicit-any\ntype AnyTraceableFunction = TraceableFunction<(...any: any[]) => any>;\n\n/**\n * Internal implementation of RunTree, which uses the\n * provided callback manager instead of the internal LangSmith client.\n *\n * The goal of this class is to ensure seamless interop when intergrated\n * with other Runnables.\n */\nclass CallbackManagerRunTree extends RunTree {\n  callbackManager: CallbackManager;\n\n  activeCallbackManager: CallbackManagerForChainRun | undefined = undefined;\n\n  constructor(config: RunTreeConfig, callbackManager: CallbackManager) {\n    super(config);\n\n    this.callbackManager = callbackManager;\n  }\n\n  createChild(config: RunTreeConfig): CallbackManagerRunTree {\n    const child = new CallbackManagerRunTree(\n      {\n        ...config,\n        parent_run: this,\n        project_name: this.project_name,\n        client: this.client,\n      },\n      this.activeCallbackManager?.getChild() ?? this.callbackManager\n    );\n    this.child_runs.push(child);\n    return child;\n  }\n\n  async postRun(): Promise<void> {\n    // how it is translated in comparison to basic RunTree?\n    this.activeCallbackManager = await this.callbackManager.handleChainStart(\n      typeof this.serialized !== \"object\" &&\n        this.serialized != null &&\n        \"lc\" in this.serialized\n        ? this.serialized\n        : {\n            id: [\"langchain\", \"smith\", \"CallbackManagerRunTree\"],\n            lc: 1,\n            type: \"not_implemented\",\n          },\n      this.inputs,\n      this.id,\n      this.run_type,\n      undefined,\n      undefined,\n      this.name\n    );\n  }\n\n  async patchRun(): Promise<void> {\n    if (this.error) {\n      await this.activeCallbackManager?.handleChainError(\n        this.error,\n        this.id,\n        this.parent_run?.id,\n        undefined,\n        undefined\n      );\n    } else {\n      await this.activeCallbackManager?.handleChainEnd(\n        this.outputs ?? {},\n        this.id,\n        this.parent_run?.id,\n        undefined,\n        undefined\n      );\n    }\n  }\n}\n\nclass RunnableTraceable<RunInput, RunOutput> extends Runnable<\n  RunInput,\n  RunOutput\n> {\n  lc_serializable = false;\n\n  lc_namespace = [\"langchain_core\", \"runnables\"];\n\n  protected func: AnyTraceableFunction;\n\n  constructor(fields: { func: AnyTraceableFunction }) {\n    super(fields);\n\n    if (!isLangsmithTraceableFunction(fields.func)) {\n      throw new Error(\n        \"RunnableTraceable requires a function that is wrapped in traceable higher-order function\"\n      );\n    }\n\n    this.func = fields.func;\n  }\n\n  async invoke(input: RunInput, options?: Partial<RunnableConfig>) {\n    const [config] = this._getOptionsList(options ?? {}, 1);\n    const callbackManager = await getCallbackManagerForConfig(config);\n\n    const partialConfig =\n      \"langsmith:traceable\" in this.func\n        ? (this.func[\"langsmith:traceable\"] as RunTreeConfig)\n        : { name: \"<lambda>\" };\n\n    if (!callbackManager) throw new Error(\"CallbackManager not found\");\n    const runTree = new CallbackManagerRunTree(\n      {\n        ...partialConfig,\n        parent_run: callbackManager?._parentRunId\n          ? new RunTree({ name: \"<parent>\", id: callbackManager?._parentRunId })\n          : undefined,\n      },\n      callbackManager\n    );\n\n    if (\n      typeof input === \"object\" &&\n      input != null &&\n      Object.keys(input).length === 1\n    ) {\n      if (\"args\" in input && Array.isArray(input)) {\n        return (await this.func(runTree, ...input)) as RunOutput;\n      }\n\n      if (\n        \"input\" in input &&\n        !(\n          typeof input === \"object\" &&\n          input != null &&\n          !Array.isArray(input) &&\n          // oxlint-disable-next-line no-instanceof/no-instanceof\n          !(input instanceof Date)\n        )\n      ) {\n        try {\n          return (await this.func(runTree, input.input)) as RunOutput;\n        } catch {\n          return (await this.func(runTree, input)) as RunOutput;\n        }\n      }\n    }\n\n    return (await this.func(runTree, input)) as RunOutput;\n  }\n}\n\n/**\n * Wraps an off-the-shelf evaluator (loaded using loadEvaluator; of EvaluatorType[T])\n * and composes with a prepareData function so the user can prepare the trace and\n * dataset data for the evaluator.\n */\nclass PreparedRunEvaluator implements RunEvaluator {\n  evaluator: LLMStringEvaluator;\n\n  formatEvaluatorInputs: EvaluatorInputFormatter;\n\n  isStringEvaluator: boolean;\n\n  evaluationName: string;\n\n  constructor(\n    evaluator: LLMStringEvaluator,\n    evaluationName: string,\n    formatEvaluatorInputs: EvaluatorInputFormatter\n  ) {\n    this.evaluator = evaluator;\n    this.isStringEvaluator = typeof evaluator?.evaluateStrings === \"function\";\n    this.evaluationName = evaluationName;\n    this.formatEvaluatorInputs = formatEvaluatorInputs;\n  }\n\n  static async fromEvalConfig(\n    config: EvalConfig | keyof EvaluatorType\n  ): Promise<PreparedRunEvaluator> {\n    const evaluatorType =\n      typeof config === \"string\" ? config : config.evaluatorType;\n    const evalConfig = typeof config === \"string\" ? ({} as EvalConfig) : config;\n    const evaluator = await loadEvaluator(evaluatorType, evalConfig);\n    const feedbackKey = evalConfig?.feedbackKey ?? evaluator?.evaluationName;\n    if (!isLLMStringEvaluator(evaluator)) {\n      throw new Error(\n        `Evaluator of type ${evaluatorType} not yet supported. ` +\n          \"Please use a string evaluator, or implement your \" +\n          \"evaluation logic as a custom evaluator.\"\n      );\n    }\n    if (!feedbackKey) {\n      throw new Error(\n        `Evaluator of type ${evaluatorType} must have an evaluationName` +\n          ` or feedbackKey. Please manually provide a feedbackKey in the EvalConfig.`\n      );\n    }\n    return new PreparedRunEvaluator(\n      evaluator as LLMStringEvaluator,\n      feedbackKey,\n      evalConfig?.formatEvaluatorInputs\n    );\n  }\n\n  /**\n   * Evaluates a run with an optional example and returns the evaluation result.\n   * @param run The run to evaluate.\n   * @param example The optional example to use for evaluation.\n   * @returns A promise that extracts to the evaluation result.\n   */\n  async evaluateRun(run: Run, example?: Example): Promise<EvaluationResult> {\n    const { prediction, input, reference } = this.formatEvaluatorInputs({\n      rawInput: run.inputs,\n      rawPrediction: run.outputs,\n      rawReferenceOutput: example?.outputs,\n      run,\n    });\n    const extractor = new SingleRunIdExtractor();\n    const tracer = new LangChainTracer({ projectName: \"evaluators\" });\n    if (this.isStringEvaluator) {\n      const evalResult = await this.evaluator.evaluateStrings(\n        {\n          prediction: prediction as string,\n          reference: reference as string,\n          input: input as string,\n        },\n        {\n          callbacks: [extractor, tracer],\n        }\n      );\n      const runId = await extractor.extract();\n      return {\n        key: this.evaluationName,\n        comment: evalResult?.reasoning,\n        sourceRunId: runId,\n        ...evalResult,\n      };\n    }\n    throw new Error(\n      \"Evaluator not yet supported. \" +\n        \"Please use a string evaluator, or implement your \" +\n        \"evaluation logic as a custom evaluator.\"\n    );\n  }\n}\n\nclass LoadedEvalConfig {\n  constructor(public evaluators: (RunEvaluator | DynamicRunEvaluator)[]) {}\n\n  static async fromRunEvalConfig(\n    config: RunEvalConfig<keyof EvaluatorType>\n  ): Promise<LoadedEvalConfig> {\n    // Custom evaluators are applied \"as-is\"\n    const customEvaluators = (\n      config?.customEvaluators ?? config.evaluators?.filter(isCustomEvaluator)\n    )?.map((evaluator) => {\n      if (typeof evaluator === \"function\") {\n        return new DynamicRunEvaluator(evaluator);\n      } else {\n        return evaluator;\n      }\n    });\n\n    const offTheShelfEvaluators = await Promise.all(\n      config?.evaluators\n        ?.filter(isOffTheShelfEvaluator)\n        ?.map(\n          async (evaluator) =>\n            await PreparedRunEvaluator.fromEvalConfig(evaluator)\n        ) ?? []\n    );\n    return new LoadedEvalConfig(\n      (customEvaluators ?? []).concat(offTheShelfEvaluators ?? [])\n    );\n  }\n}\n\nexport interface RunOnDatasetParams extends Omit<\n  RunEvalConfig,\n  \"customEvaluators\"\n> {\n  /**\n   * Name of the project for logging and tracking.\n   */\n  projectName?: string;\n\n  /**\n   * Additional metadata for the project.\n   */\n  projectMetadata?: Record<string, unknown>;\n\n  /**\n   * Client instance for LangSmith service interaction.\n   */\n  client?: Client;\n\n  /**\n   * Maximum concurrency level for dataset processing.\n   */\n  maxConcurrency?: number;\n\n  /**\n   * @deprecated Pass keys directly to the RunOnDatasetParams instead\n   */\n  evaluationConfig?: RunEvalConfig;\n}\n\n/**\n * Internals expect a constructor () -> Runnable. This function wraps/coerces\n * the provided LangChain object, custom function, or factory function into\n * a constructor of a runnable.\n * @param modelOrFactory The model or factory to create a wrapped model from.\n * @returns A function that returns the wrapped model.\n * @throws Error if the modelOrFactory is invalid.\n */\nconst createWrappedModel = async (modelOrFactory: ChainOrFactory) => {\n  if (Runnable.isRunnable(modelOrFactory)) {\n    return () => modelOrFactory;\n  }\n  if (typeof modelOrFactory === \"function\") {\n    if (isLangsmithTraceableFunction(modelOrFactory)) {\n      const wrappedModel = new RunnableTraceable({ func: modelOrFactory });\n      return () => wrappedModel;\n    }\n\n    try {\n      // If it works with no arguments, assume it's a factory\n      let res = (modelOrFactory as () => Runnable)();\n      if (\n        res &&\n        typeof (res as unknown as Promise<Runnable>).then === \"function\"\n      ) {\n        res = await res;\n      }\n      return modelOrFactory as () => Runnable;\n    } catch {\n      // Otherwise, it's a custom UDF, and we'll wrap\n      // the function in a lambda\n      const wrappedModel = new RunnableLambda({ func: modelOrFactory });\n      return () => wrappedModel;\n    }\n  }\n  throw new Error(\"Invalid modelOrFactory\");\n};\n\nconst loadExamples = async ({\n  datasetName,\n  client,\n  projectName,\n}: {\n  datasetName: string;\n  client: Client;\n  projectName: string;\n  maxConcurrency: number;\n}) => {\n  const exampleIterator = client.listExamples({ datasetName });\n  const configs: RunnableConfig[] = [];\n  const runExtractors = [];\n  const examples = [];\n  for await (const example of exampleIterator) {\n    const runExtractor = new SingleRunExtractor();\n    configs.push({\n      callbacks: [\n        new LangChainTracer({ exampleId: example.id, projectName }),\n        runExtractor,\n      ],\n    });\n    examples.push(example);\n    runExtractors.push(runExtractor);\n  }\n  return {\n    configs,\n    examples,\n    runExtractors,\n  };\n};\n\nconst applyEvaluators = async ({\n  evaluation,\n  runs,\n  examples,\n  client,\n  maxConcurrency,\n}: {\n  evaluation: LoadedEvalConfig;\n  runs: Run[];\n  examples: Example[];\n  client: Client;\n  maxConcurrency: number;\n}): Promise<{\n  [key: string]: {\n    execution_time?: number;\n    run_id: string;\n    feedback: Feedback[];\n  };\n}> => {\n  // TODO: Parallelize and/or put in callbacks to speed up evals.\n  const { evaluators } = evaluation;\n  const progress = new ProgressBar({\n    total: examples.length,\n    format: \"Running Evaluators: {bar} {percentage}% | {value}/{total}\\n\",\n  });\n  const caller = new AsyncCaller({\n    maxConcurrency,\n  });\n  const requests = runs.map(\n    async (\n      run,\n      i\n    ): Promise<{\n      run_id: string;\n      execution_time?: number;\n      feedback: Feedback[];\n    }> =>\n      caller.call(async () => {\n        const evaluatorResults = await Promise.allSettled(\n          evaluators.map((evaluator) =>\n            client.evaluateRun(run, evaluator, {\n              referenceExample: examples[i],\n              loadChildRuns: false,\n            })\n          )\n        );\n        progress.increment();\n        return {\n          execution_time:\n            run?.end_time && run.start_time\n              ? new Date(run.end_time).getTime() -\n                new Date(run.start_time).getTime()\n              : undefined,\n          feedback: evaluatorResults.map((evalResult) =>\n            evalResult.status === \"fulfilled\"\n              ? evalResult.value\n              : evalResult.reason\n          ),\n          run_id: run.id,\n        };\n      })\n  );\n  const results = await Promise.all(requests);\n\n  return results.reduce(\n    (acc, result, i) => ({\n      ...acc,\n      [examples[i].id]: result,\n    }),\n    {}\n  );\n};\n\nexport type EvalResults = {\n  projectName: string;\n  results: {\n    [key: string]: {\n      execution_time?: number;\n      run_id: string;\n      feedback: Feedback[];\n    };\n  };\n};\n\nconst getExamplesInputs = (\n  examples: Example[],\n  chainOrFactory: ChainOrFactory,\n  dataType?: DataType\n) => {\n  if (dataType === \"chat\") {\n    // For some batty reason, we store the chat dataset differently.\n    // { type: \"system\", data: { content: inputs.input } },\n    // But we need to create AIMesage, SystemMessage, etc.\n    return examples.map(({ inputs }) =>\n      mapStoredMessagesToChatMessages(inputs.input)\n    );\n  }\n  // If it's a language model and ALL example inputs have a single value,\n  // then we can be friendly and flatten the inputs to a list of strings.\n  const isLanguageModel =\n    typeof chainOrFactory === \"object\" &&\n    typeof (chainOrFactory as BaseLanguageModel)._llmType === \"function\";\n  if (\n    isLanguageModel &&\n    examples.every(({ inputs }) => Object.keys(inputs).length === 1)\n  ) {\n    return examples.map(({ inputs }) => Object.values(inputs)[0]);\n  }\n  return examples.map(({ inputs }) => inputs);\n};\n\n/**\n * Evaluates a given model or chain against a specified LangSmith dataset.\n *\n * This function fetches example records from the specified dataset,\n * runs the model or chain against each example, and returns the evaluation\n * results.\n *\n * @param chainOrFactory - A model or factory/constructor function to be evaluated. It can be a\n * Runnable instance, a factory function that returns a Runnable, or a user-defined\n * function or factory.\n *\n * @param datasetName - The name of the dataset against which the evaluation will be\n * performed. This dataset should already be defined and contain the relevant data\n * for evaluation.\n *\n * @param options - (Optional) Additional parameters for the evaluation process:\n *   - `evaluators` (RunEvalType[]): Evaluators to apply to a dataset run.\n *   - `formatEvaluatorInputs` (EvaluatorInputFormatter): Convert the evaluation data into formats that can be used by the evaluator.\n *   - `projectName` (string): Name of the project for logging and tracking.\n *   - `projectMetadata` (Record<string, unknown>): Additional metadata for the project.\n *   - `client` (Client): Client instance for LangSmith service interaction.\n *   - `maxConcurrency` (number): Maximum concurrency level for dataset processing.\n *\n * @returns A promise that resolves to an `EvalResults` object. This object includes\n * detailed results of the evaluation, such as execution time, run IDs, and feedback\n * for each entry in the dataset.\n *\n * @example\n * ```typescript\n * // Example usage for evaluating a model on a dataset\n * async function evaluateModel() {\n *   const chain = /* ...create your model or chain...*\\//\n *   const datasetName = 'example-dataset';\n *   const client = new Client(/* ...config... *\\//);\n *\n *   const results = await runOnDataset(chain, datasetName, {\n *     evaluators: [/* ...evaluators... *\\//],\n *     client,\n *   });\n *\n *   console.log('Evaluation Results:', results);\n * }\n *\n * evaluateModel();\n * ```\n * In this example, `runOnDataset` is used to evaluate a language model (or a chain of models) against\n * a dataset named 'example-dataset'. The evaluation process is configured using `RunOnDatasetParams[\"evaluators\"]`, which can\n * include both standard and custom evaluators. The `Client` instance is used to interact with LangChain services.\n * The function returns the evaluation results, which can be logged or further processed as needed.\n */\n\nexport async function runOnDataset(\n  chainOrFactory: ChainOrFactory,\n  datasetName: string,\n  options?: RunOnDatasetParams\n) {\n  const {\n    projectName,\n    projectMetadata,\n    client,\n    maxConcurrency,\n  }: RunOnDatasetParams = options ?? {};\n\n  const evaluationConfig: RunEvalConfig | undefined =\n    options?.evaluationConfig ??\n    (options?.evaluators != null\n      ? {\n          evaluators: options.evaluators,\n          formatEvaluatorInputs: options.formatEvaluatorInputs,\n        }\n      : undefined);\n\n  const wrappedModel = await createWrappedModel(chainOrFactory);\n  const testClient = client ?? new Client();\n  const testProjectName = projectName ?? randomName();\n  const dataset = await testClient.readDataset({ datasetName });\n  const datasetId = dataset.id;\n  const testConcurrency = maxConcurrency ?? 5;\n  const { configs, examples, runExtractors } = await loadExamples({\n    datasetName,\n    client: testClient,\n    projectName: testProjectName,\n    maxConcurrency: testConcurrency,\n  });\n\n  await testClient.createProject({\n    projectName: testProjectName,\n    referenceDatasetId: datasetId,\n    projectExtra: { metadata: { ...projectMetadata } },\n  });\n  const wrappedRunnable: Runnable = new RunnableLambda({\n    func: wrappedModel,\n  }).withConfig({ runName: \"evaluationRun\" });\n  const runInputs = getExamplesInputs(\n    examples,\n    chainOrFactory,\n    dataset.data_type\n  );\n  const progress = new ProgressBar({\n    total: runInputs.length,\n    format: \"Predicting: {bar} {percentage}% | {value}/{total}\",\n  });\n  // TODO: Collect the runs as well.\n  await wrappedRunnable\n    .withListeners({\n      onEnd: () => progress.increment(),\n    })\n    // TODO: Insert evaluation inline for immediate feedback.\n    .batch(runInputs, configs, {\n      maxConcurrency,\n      returnExceptions: true,\n    });\n\n  progress.complete();\n  const runs: Run[] = [];\n  for (let i = 0; i < examples.length; i += 1) {\n    runs.push(await runExtractors[i].extract());\n  }\n  let evalResults: Record<\n    string,\n    { run_id: string; execution_time?: number; feedback: Feedback[] }\n  > = {};\n  if (evaluationConfig) {\n    const loadedEvalConfig =\n      await LoadedEvalConfig.fromRunEvalConfig(evaluationConfig);\n    evalResults = await applyEvaluators({\n      evaluation: loadedEvalConfig,\n      runs,\n      examples,\n      client: testClient,\n      maxConcurrency: testConcurrency,\n    });\n  }\n  const results: EvalResults = {\n    projectName: testProjectName,\n    results: evalResults ?? {},\n  };\n  return results;\n}\n\nfunction isLangsmithTraceableFunction(x: unknown): x is AnyTraceableFunction {\n  return typeof x === \"function\" && \"langsmith:traceable\" in x;\n}\n"],"mappings":";;;;;;;;;;;AAsDA,IAAM,uBAAN,MAA2B;CACzB;CAEA;CAEA,cAAc;EACZ,KAAK,eAAe,IAAI,SAAiB,YAAY;GACnD,KAAK,uBAAuB;EAC9B,CAAC;CACH;CAEA,oBACE,QACA,SACA,UACG;EACH,KAAK,qBAAqB,KAAK;CACjC;CAEA,MAAM,UAA2B;EAC/B,OAAO,KAAK;CACd;AACF;AAEA,IAAM,qBAAN,cAAiCA,6BAAAA,WAAW;CAC1C;CAEA;;CAGA,OAAO;CAEP,cAAc;EACZ,MAAM;EACN,KAAK,aAAa,IAAI,SAAc,YAAY;GAC9C,KAAK,qBAAqB;EAC5B,CAAC;CACH;CAEA,MAAM,WAAW,KAAU;EACzB,KAAK,mBAAmB,GAAG;CAC7B;CAEA,MAAM,UAAwB;EAC5B,OAAO,KAAK;CACd;AACF;;;;AAKA,IAAM,sBAAN,MAAkD;CAChD;CAEA,YAAY,WAA6B;EACvC,KAAK,YAAY,IAAIC,0BAAAA,eAAe,EAAE,MAAM,UAAU,CAAC;CACzD;;;;;;;CAQA,MAAM,YAAY,KAAU,SAA8C;EACxE,MAAM,YAAY,IAAI,qBAAqB;EAC3C,MAAM,SAAS,IAAIC,yCAAAA,gBAAgB,EAAE,aAAa,aAAa,CAAC;EAChE,MAAM,SAAS,MAAM,KAAK,UAAU,OAClC;GACE;GACA;GACA,OAAO,IAAI;GACX,YAAY,IAAI;GAChB,WAAW,SAAS;EACtB,GACA,EACE,WAAW,CAAC,WAAW,MAAM,EAC/B,CACF;EAEA,OAAO;GACL,aAAa,MAFK,UAAU,QAAQ;GAGpC,GAAG;EACL;CACF;AACF;AAGA,SAAS,qBAAqB,WAAiD;CAC7E,OAAO,aAAa,OAAO,UAAU,oBAAoB;AAC3D;;;;;;;;AAYA,IAAM,yBAAN,MAAM,+BAA+BC,UAAAA,QAAQ;CAC3C;CAEA,wBAAgE,KAAA;CAEhE,YAAY,QAAuB,iBAAkC;EACnE,MAAM,MAAM;EAEZ,KAAK,kBAAkB;CACzB;CAEA,YAAY,QAA+C;EACzD,MAAM,QAAQ,IAAI,uBAChB;GACE,GAAG;GACH,YAAY;GACZ,cAAc,KAAK;GACnB,QAAQ,KAAK;EACf,GACA,KAAK,uBAAuB,SAAS,KAAK,KAAK,eACjD;EACA,KAAK,WAAW,KAAK,KAAK;EAC1B,OAAO;CACT;CAEA,MAAM,UAAyB;EAE7B,KAAK,wBAAwB,MAAM,KAAK,gBAAgB,iBACtD,OAAO,KAAK,eAAe,YACzB,KAAK,cAAc,QACnB,QAAQ,KAAK,aACX,KAAK,aACL;GACE,IAAI;IAAC;IAAa;IAAS;GAAwB;GACnD,IAAI;GACJ,MAAM;EACR,GACJ,KAAK,QACL,KAAK,IACL,KAAK,UACL,KAAA,GACA,KAAA,GACA,KAAK,IACP;CACF;CAEA,MAAM,WAA0B;EAC9B,IAAI,KAAK,OACP,MAAM,KAAK,uBAAuB,iBAChC,KAAK,OACL,KAAK,IACL,KAAK,YAAY,IACjB,KAAA,GACA,KAAA,CACF;OAEA,MAAM,KAAK,uBAAuB,eAChC,KAAK,WAAW,CAAC,GACjB,KAAK,IACL,KAAK,YAAY,IACjB,KAAA,GACA,KAAA,CACF;CAEJ;AACF;AAEA,IAAM,oBAAN,cAAqDC,0BAAAA,SAGnD;CACA,kBAAkB;CAElB,eAAe,CAAC,kBAAkB,WAAW;CAE7C;CAEA,YAAY,QAAwC;EAClD,MAAM,MAAM;EAEZ,IAAI,CAAC,6BAA6B,OAAO,IAAI,GAC3C,MAAM,IAAI,MACR,0FACF;EAGF,KAAK,OAAO,OAAO;CACrB;CAEA,MAAM,OAAO,OAAiB,SAAmC;EAC/D,MAAM,CAAC,UAAU,KAAK,gBAAgB,WAAW,CAAC,GAAG,CAAC;EACtD,MAAM,kBAAkB,OAAA,GAAA,0BAAA,4BAAA,CAAkC,MAAM;EAEhE,MAAM,gBACJ,yBAAyB,KAAK,OACzB,KAAK,KAAK,yBACX,EAAE,MAAM,WAAW;EAEzB,IAAI,CAAC,iBAAiB,MAAM,IAAI,MAAM,2BAA2B;EACjE,MAAM,UAAU,IAAI,uBAClB;GACE,GAAG;GACH,YAAY,iBAAiB,eACzB,IAAID,UAAAA,QAAQ;IAAE,MAAM;IAAY,IAAI,iBAAiB;GAAa,CAAC,IACnE,KAAA;EACN,GACA,eACF;EAEA,IACE,OAAO,UAAU,YACjB,SAAS,QACT,OAAO,KAAK,KAAK,CAAC,CAAC,WAAW,GAC9B;GACA,IAAI,UAAU,SAAS,MAAM,QAAQ,KAAK,GACxC,OAAQ,MAAM,KAAK,KAAK,SAAS,GAAG,KAAK;GAG3C,IACE,WAAW,SACX,EACE,OAAO,UAAU,YACjB,SAAS,QACT,CAAC,MAAM,QAAQ,KAAK,KAEpB,EAAE,iBAAiB,QAGrB,IAAI;IACF,OAAQ,MAAM,KAAK,KAAK,SAAS,MAAM,KAAK;GAC9C,QAAQ;IACN,OAAQ,MAAM,KAAK,KAAK,SAAS,KAAK;GACxC;EAEJ;EAEA,OAAQ,MAAM,KAAK,KAAK,SAAS,KAAK;CACxC;AACF;;;;;;AAOA,IAAM,uBAAN,MAAM,qBAA6C;CACjD;CAEA;CAEA;CAEA;CAEA,YACE,WACA,gBACA,uBACA;EACA,KAAK,YAAY;EACjB,KAAK,oBAAoB,OAAO,WAAW,oBAAoB;EAC/D,KAAK,iBAAiB;EACtB,KAAK,wBAAwB;CAC/B;CAEA,aAAa,eACX,QAC+B;EAC/B,MAAM,gBACJ,OAAO,WAAW,WAAW,SAAS,OAAO;EAC/C,MAAM,aAAa,OAAO,WAAW,WAAY,CAAC,IAAmB;EACrE,MAAM,YAAY,MAAME,eAAAA,cAAc,eAAe,UAAU;EAC/D,MAAM,cAAc,YAAY,eAAe,WAAW;EAC1D,IAAI,CAAC,qBAAqB,SAAS,GACjC,MAAM,IAAI,MACR,qBAAqB,cAAc,6GAGrC;EAEF,IAAI,CAAC,aACH,MAAM,IAAI,MACR,qBAAqB,cAAc,sGAErC;EAEF,OAAO,IAAI,qBACT,WACA,aACA,YAAY,qBACd;CACF;;;;;;;CAQA,MAAM,YAAY,KAAU,SAA8C;EACxE,MAAM,EAAE,YAAY,OAAO,cAAc,KAAK,sBAAsB;GAClE,UAAU,IAAI;GACd,eAAe,IAAI;GACnB,oBAAoB,SAAS;GAC7B;EACF,CAAC;EACD,MAAM,YAAY,IAAI,qBAAqB;EAC3C,MAAM,SAAS,IAAIH,yCAAAA,gBAAgB,EAAE,aAAa,aAAa,CAAC;EAChE,IAAI,KAAK,mBAAmB;GAC1B,MAAM,aAAa,MAAM,KAAK,UAAU,gBACtC;IACc;IACD;IACJ;GACT,GACA,EACE,WAAW,CAAC,WAAW,MAAM,EAC/B,CACF;GACA,MAAM,QAAQ,MAAM,UAAU,QAAQ;GACtC,OAAO;IACL,KAAK,KAAK;IACV,SAAS,YAAY;IACrB,aAAa;IACb,GAAG;GACL;EACF;EACA,MAAM,IAAI,MACR,uHAGF;CACF;AACF;AAEA,IAAM,mBAAN,MAAM,iBAAiB;CACF;CAAnB,YAAY,YAA2D;EAApD,KAAA,aAAA;CAAqD;CAExE,aAAa,kBACX,QAC2B;EAE3B,MAAM,oBACJ,QAAQ,oBAAoB,OAAO,YAAY,OAAOI,eAAAA,iBAAiB,EAAA,EACtE,KAAK,cAAc;GACpB,IAAI,OAAO,cAAc,YACvB,OAAO,IAAI,oBAAoB,SAAS;QAExC,OAAO;EAEX,CAAC;EAED,MAAM,wBAAwB,MAAM,QAAQ,IAC1C,QAAQ,YACJ,OAAOC,eAAAA,sBAAsB,CAAC,EAC9B,IACA,OAAO,cACL,MAAM,qBAAqB,eAAe,SAAS,CACvD,KAAK,CAAC,CACV;EACA,OAAO,IAAI,kBACR,oBAAoB,CAAC,EAAA,CAAG,OAAO,yBAAyB,CAAC,CAAC,CAC7D;CACF;AACF;;;;;;;;;AAwCA,MAAM,qBAAqB,OAAO,mBAAmC;CACnE,IAAIH,0BAAAA,SAAS,WAAW,cAAc,GACpC,aAAa;CAEf,IAAI,OAAO,mBAAmB,YAAY;EACxC,IAAI,6BAA6B,cAAc,GAAG;GAChD,MAAM,eAAe,IAAI,kBAAkB,EAAE,MAAM,eAAe,CAAC;GACnE,aAAa;EACf;EAEA,IAAI;GAEF,IAAI,MAAO,eAAkC;GAC7C,IACE,OACA,OAAQ,IAAqC,SAAS,YAEtD,MAAM,MAAM;GAEd,OAAO;EACT,QAAQ;GAGN,MAAM,eAAe,IAAIH,0BAAAA,eAAe,EAAE,MAAM,eAAe,CAAC;GAChE,aAAa;EACf;CACF;CACA,MAAM,IAAI,MAAM,wBAAwB;AAC1C;AAEA,MAAM,eAAe,OAAO,EAC1B,aACA,QACA,kBAMI;CACJ,MAAM,kBAAkB,OAAO,aAAa,EAAE,YAAY,CAAC;CAC3D,MAAM,UAA4B,CAAC;CACnC,MAAM,gBAAgB,CAAC;CACvB,MAAM,WAAW,CAAC;CAClB,WAAW,MAAM,WAAW,iBAAiB;EAC3C,MAAM,eAAe,IAAI,mBAAmB;EAC5C,QAAQ,KAAK,EACX,WAAW,CACT,IAAIC,yCAAAA,gBAAgB;GAAE,WAAW,QAAQ;GAAI;EAAY,CAAC,GAC1D,YACF,EACF,CAAC;EACD,SAAS,KAAK,OAAO;EACrB,cAAc,KAAK,YAAY;CACjC;CACA,OAAO;EACL;EACA;EACA;CACF;AACF;AAEA,MAAM,kBAAkB,OAAO,EAC7B,YACA,MACA,UACA,QACA,qBAaI;CAEJ,MAAM,EAAE,eAAe;CACvB,MAAM,WAAW,IAAIM,iBAAAA,YAAY;EAC/B,OAAO,SAAS;EAChB,QAAQ;CACV,CAAC;CACD,MAAM,SAAS,IAAIC,mCAAAA,YAAY,EAC7B,eACF,CAAC;CACD,MAAM,WAAW,KAAK,IACpB,OACE,KACA,MAMA,OAAO,KAAK,YAAY;EACtB,MAAM,mBAAmB,MAAM,QAAQ,WACrC,WAAW,KAAK,cACd,OAAO,YAAY,KAAK,WAAW;GACjC,kBAAkB,SAAS;GAC3B,eAAe;EACjB,CAAC,CACH,CACF;EACA,SAAS,UAAU;EACnB,OAAO;GACL,gBACE,KAAK,YAAY,IAAI,aACjB,IAAI,KAAK,IAAI,QAAQ,CAAC,CAAC,QAAQ,IAC/B,IAAI,KAAK,IAAI,UAAU,CAAC,CAAC,QAAQ,IACjC,KAAA;GACN,UAAU,iBAAiB,KAAK,eAC9B,WAAW,WAAW,cAClB,WAAW,QACX,WAAW,MACjB;GACA,QAAQ,IAAI;EACd;CACF,CAAC,CACL;CAGA,QAAO,MAFe,QAAQ,IAAI,QAAQ,EAAA,CAE3B,QACZ,KAAK,QAAQ,OAAO;EACnB,GAAG;GACF,SAAS,EAAE,CAAC,KAAK;CACpB,IACA,CAAC,CACH;AACF;AAaA,MAAM,qBACJ,UACA,gBACA,aACG;CACH,IAAI,aAAa,QAIf,OAAO,SAAS,KAAK,EAAE,cAAA,GAAA,yBAAA,gCAAA,CACW,OAAO,KAAK,CAC9C;CAOF,IAFE,OAAO,mBAAmB,YAC1B,OAAQ,eAAqC,aAAa,cAG1D,SAAS,OAAO,EAAE,aAAa,OAAO,KAAK,MAAM,CAAC,CAAC,WAAW,CAAC,GAE/D,OAAO,SAAS,KAAK,EAAE,aAAa,OAAO,OAAO,MAAM,CAAC,CAAC,EAAE;CAE9D,OAAO,SAAS,KAAK,EAAE,aAAa,MAAM;AAC5C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAqDA,eAAsB,aACpB,gBACA,aACA,SACA;CACA,MAAM,EACJ,aACA,iBACA,QACA,mBACsB,WAAW,CAAC;CAEpC,MAAM,mBACJ,SAAS,qBACR,SAAS,cAAc,OACpB;EACE,YAAY,QAAQ;EACpB,uBAAuB,QAAQ;CACjC,IACA,KAAA;CAEN,MAAM,eAAe,MAAM,mBAAmB,cAAc;CAC5D,MAAM,aAAa,UAAU,IAAIC,UAAAA,OAAO;CACxC,MAAM,kBAAkB,eAAeC,wBAAAA,WAAW;CAClD,MAAM,UAAU,MAAM,WAAW,YAAY,EAAE,YAAY,CAAC;CAC5D,MAAM,YAAY,QAAQ;CAC1B,MAAM,kBAAkB,kBAAkB;CAC1C,MAAM,EAAE,SAAS,UAAU,kBAAkB,MAAM,aAAa;EAC9D;EACA,QAAQ;EACR,aAAa;EACb,gBAAgB;CAClB,CAAC;CAED,MAAM,WAAW,cAAc;EAC7B,aAAa;EACb,oBAAoB;EACpB,cAAc,EAAE,UAAU,EAAE,GAAG,gBAAgB,EAAE;CACnD,CAAC;CACD,MAAM,kBAA4B,IAAIV,0BAAAA,eAAe,EACnD,MAAM,aACR,CAAC,CAAC,CAAC,WAAW,EAAE,SAAS,gBAAgB,CAAC;CAC1C,MAAM,YAAY,kBAChB,UACA,gBACA,QAAQ,SACV;CACA,MAAM,WAAW,IAAIO,iBAAAA,YAAY;EAC/B,OAAO,UAAU;EACjB,QAAQ;CACV,CAAC;CAED,MAAM,gBACH,cAAc,EACb,aAAa,SAAS,UAAU,EAClC,CAAC,CAAC,CAED,MAAM,WAAW,SAAS;EACzB;EACA,kBAAkB;CACpB,CAAC;CAEH,SAAS,SAAS;CAClB,MAAM,OAAc,CAAC;CACrB,KAAK,IAAI,IAAI,GAAG,IAAI,SAAS,QAAQ,KAAK,GACxC,KAAK,KAAK,MAAM,cAAc,EAAE,CAAC,QAAQ,CAAC;CAE5C,IAAI,cAGA,CAAC;CACL,IAAI,kBAAkB;EACpB,MAAM,mBACJ,MAAM,iBAAiB,kBAAkB,gBAAgB;EAC3D,cAAc,MAAM,gBAAgB;GAClC,YAAY;GACZ;GACA;GACA,QAAQ;GACR,gBAAgB;EAClB,CAAC;CACH;CAKA,OAAO;EAHL,aAAa;EACb,SAAS,eAAe,CAAC;CAEd;AACf;AAEA,SAAS,6BAA6B,GAAuC;CAC3E,OAAO,OAAO,MAAM,cAAc,yBAAyB;AAC7D"}

package/dist/smith/runner_utils.d.cts

@@ -2,7 +2,6 @@ import { RunEvalConfig } from "./config.cjs";
 import { Runnable } from "@langchain/core/runnables";
 import { Client, Feedback } from "langsmith";
 import { TraceableFunction } from "langsmith/singletons/traceable";
-
 //#region src/smith/runner_utils.d.ts
 type ChainOrFactory = Runnable | (() => Runnable) | AnyTraceableFunction | ((obj: any) => any) | ((obj: any) => Promise<any>) | (() => (obj: unknown) => unknown) | (() => (obj: unknown) => Promise<unknown>);
 type AnyTraceableFunction = TraceableFunction<(...any: any[]) => any>;
@@ -90,5 +89,5 @@ type EvalResults = {
  */
 declare function runOnDataset(chainOrFactory: ChainOrFactory, datasetName: string, options?: RunOnDatasetParams): Promise<EvalResults>;
 //#endregion
-export { EvalResults, RunOnDatasetParams, runOnDataset };
+export { ChainOrFactory, EvalResults, RunOnDatasetParams, runOnDataset };
 //# sourceMappingURL=runner_utils.d.cts.map

package/dist/smith/runner_utils.d.cts.map

@@ -1 +1 @@
-{"version":3,"file":"runner_utils.d.cts","names":[],"sources":["../../src/smith/runner_utils.ts"],"mappings":";;;;;;KA2CY,cAAA,GACR,QAAA,UACO,QAAA,IACP,oBAAA,KAEE,GAAA,mBAEA,GAAA,UAAa,OAAA,iBACP,GAAA,iCACA,GAAA,cAAiB,OAAA;AAAA,KA+FxB,oBAAA,GAAuB,iBAAA,KAAsB,GAAA;AAAA,UAmRjC,kBAAA,SAA2B,IAAA,CAC1C,aAAA;EA5XwB;;;EAkYxB,WAAA;EA/XE;;;EAoYF,eAAA,GAAkB,MAAA;EA9XgB;;;EAmYlC,MAAA,GAAS,MAAA;EAvYL;;;EA4YJ,cAAA;EAxYU;;;EA6YV,gBAAA,GAAmB,aAAA;AAAA;AAAA,KAkJT,WAAA;EACV,WAAA;EACA,OAAA;IAAA,CACG,GAAA;MACC,cAAA;MACA,MAAA;MACA,QAAA,EAAU,QAAA;IAAA;EAAA;AAAA;;;;;;;;;;;;;;;;;AANhB;;;;;;;;;;;;;AAyFA;;;;;;;;;;;;;;;;;;;;;iBAAsB,YAAA,CACpB,cAAA,EAAgB,cAAA,EAChB,WAAA,UACA,OAAA,GAAU,kBAAA,GAAkB,OAAA,CAAA,WAAA"}
+{"version":3,"file":"runner_utils.d.cts","names":[],"sources":["../../src/smith/runner_utils.ts"],"mappings":";;;;;KA2CY,iBACR,kBACO,YACP,yBAEE,sBAEA,aAAa,wBACP,oCACA,iBAAiB;KA+FxB,uBAAuB,sBAAsB;UAmRjC,2BAA2B,KAC1C;;;;EAMA;;;;EAKA,kBAAkB;;;;EAKlB,SAAS;;;;EAKT;;;;EAKA,mBAAmB;;KAkJT;EACV;EACA;KACG;MACC;MACA;MACA,UAAU;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAmFM,aACpB,gBAAgB,gBAChB,qBACA,UAAU,qBAAkB,QAAA"}

package/dist/smith/runner_utils.d.ts

@@ -2,7 +2,6 @@ import { RunEvalConfig } from "./config.js";
 import { Runnable } from "@langchain/core/runnables";
 import { Client, Feedback } from "langsmith";
 import { TraceableFunction } from "langsmith/singletons/traceable";
-
 //#region src/smith/runner_utils.d.ts
 type ChainOrFactory = Runnable | (() => Runnable) | AnyTraceableFunction | ((obj: any) => any) | ((obj: any) => Promise<any>) | (() => (obj: unknown) => unknown) | (() => (obj: unknown) => Promise<unknown>);
 type AnyTraceableFunction = TraceableFunction<(...any: any[]) => any>;
@@ -90,5 +89,5 @@ type EvalResults = {
  */
 declare function runOnDataset(chainOrFactory: ChainOrFactory, datasetName: string, options?: RunOnDatasetParams): Promise<EvalResults>;
 //#endregion
-export { EvalResults, RunOnDatasetParams, runOnDataset };
+export { ChainOrFactory, EvalResults, RunOnDatasetParams, runOnDataset };
 //# sourceMappingURL=runner_utils.d.ts.map

package/dist/smith/runner_utils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"runner_utils.d.ts","names":[],"sources":["../../src/smith/runner_utils.ts"],"mappings":";;;;;;KA2CY,cAAA,GACR,QAAA,UACO,QAAA,IACP,oBAAA,KAEE,GAAA,mBAEA,GAAA,UAAa,OAAA,iBACP,GAAA,iCACA,GAAA,cAAiB,OAAA;AAAA,KA+FxB,oBAAA,GAAuB,iBAAA,KAAsB,GAAA;AAAA,UAmRjC,kBAAA,SAA2B,IAAA,CAC1C,aAAA;EA5XwB;;;EAkYxB,WAAA;EA/XE;;;EAoYF,eAAA,GAAkB,MAAA;EA9XgB;;;EAmYlC,MAAA,GAAS,MAAA;EAvYL;;;EA4YJ,cAAA;EAxYU;;;EA6YV,gBAAA,GAAmB,aAAA;AAAA;AAAA,KAkJT,WAAA;EACV,WAAA;EACA,OAAA;IAAA,CACG,GAAA;MACC,cAAA;MACA,MAAA;MACA,QAAA,EAAU,QAAA;IAAA;EAAA;AAAA;;;;;;;;;;;;;;;;;AANhB;;;;;;;;;;;;;AAyFA;;;;;;;;;;;;;;;;;;;;;iBAAsB,YAAA,CACpB,cAAA,EAAgB,cAAA,EAChB,WAAA,UACA,OAAA,GAAU,kBAAA,GAAkB,OAAA,CAAA,WAAA"}
+{"version":3,"file":"runner_utils.d.ts","names":[],"sources":["../../src/smith/runner_utils.ts"],"mappings":";;;;;KA2CY,iBACR,kBACO,YACP,yBAEE,sBAEA,aAAa,wBACP,oCACA,iBAAiB;KA+FxB,uBAAuB,sBAAsB;UAmRjC,2BAA2B,KAC1C;;;;EAMA;;;;EAKA,kBAAkB;;;;EAKlB,SAAS;;;;EAKT;;;;EAKA,mBAAmB;;KAkJT;EACV;EACA;KACG;MACC;MACA;MACA,UAAU;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAmFM,aACpB,gBAAgB,gBAChB,qBACA,UAAU,qBAAkB,QAAA"}

package/dist/smith/runner_utils.js

@@ -205,6 +205,7 @@ var PreparedRunEvaluator = class PreparedRunEvaluator {
 	}
 };
 var LoadedEvalConfig = class LoadedEvalConfig {
+	evaluators;
 	constructor(evaluators) {
 		this.evaluators = evaluators;
 	}
@@ -379,13 +380,16 @@ async function runOnDataset(chainOrFactory, datasetName, options) {
 	const runs = [];
 	for (let i = 0; i < examples.length; i += 1) runs.push(await runExtractors[i].extract());
 	let evalResults = {};
-	if (evaluationConfig) evalResults = await applyEvaluators({
-		evaluation: await LoadedEvalConfig.fromRunEvalConfig(evaluationConfig),
-		runs,
-		examples,
-		client: testClient,
-		maxConcurrency: testConcurrency
-	});
+	if (evaluationConfig) {
+		const loadedEvalConfig = await LoadedEvalConfig.fromRunEvalConfig(evaluationConfig);
+		evalResults = await applyEvaluators({
+			evaluation: loadedEvalConfig,
+			runs,
+			examples,
+			client: testClient,
+			maxConcurrency: testConcurrency
+		});
+	}
 	return {
 		projectName: testProjectName,
 		results: evalResults ?? {}

package/dist/smith/runner_utils.js.map

@@ -1 +1 @@
-{"version":3,"file":"runner_utils.js","names":[],"sources":["../../src/smith/runner_utils.ts"],"sourcesContent":["import { BaseLanguageModel } from \"@langchain/core/language_models/base\";\nimport { Serialized } from \"@langchain/core/load/serializable\";\nimport { mapStoredMessagesToChatMessages } from \"@langchain/core/messages\";\nimport {\n  Runnable,\n  RunnableConfig,\n  RunnableLambda,\n  getCallbackManagerForConfig,\n} from \"@langchain/core/runnables\";\nimport { LangChainTracer } from \"@langchain/core/tracers/tracer_langchain\";\nimport { BaseTracer } from \"@langchain/core/tracers/base\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { AsyncCaller } from \"@langchain/core/utils/async_caller\";\nimport type {\n  CallbackManager,\n  CallbackManagerForChainRun,\n} from \"@langchain/core/callbacks/manager\";\nimport {\n  Client,\n  Example,\n  Feedback,\n  Run,\n  RunTree,\n  RunTreeConfig,\n} from \"langsmith\";\nimport { EvaluationResult, RunEvaluator } from \"langsmith/evaluation\";\nimport { DataType } from \"langsmith/schemas\";\nimport type { TraceableFunction } from \"langsmith/singletons/traceable\";\nimport { LLMStringEvaluator } from \"../evaluation/base.js\";\nimport { loadEvaluator } from \"../evaluation/loader.js\";\nimport { EvaluatorType } from \"../evaluation/types.js\";\nimport {\n  isOffTheShelfEvaluator,\n  type DynamicRunEvaluatorParams,\n  type EvalConfig,\n  type EvaluatorInputFormatter,\n  type RunEvalConfig,\n  type RunEvaluatorLike,\n  isCustomEvaluator,\n} from \"./config.js\";\nimport { randomName } from \"./name_generation.js\";\nimport { ProgressBar } from \"./progress.js\";\n\nexport type ChainOrFactory =\n  | Runnable\n  | (() => Runnable)\n  | AnyTraceableFunction\n  // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n  | ((obj: any) => any)\n  // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n  | ((obj: any) => Promise<any>)\n  | (() => (obj: unknown) => unknown)\n  | (() => (obj: unknown) => Promise<unknown>);\n\nclass SingleRunIdExtractor {\n  runIdPromiseResolver: (runId: string) => void;\n\n  runIdPromise: Promise<string>;\n\n  constructor() {\n    this.runIdPromise = new Promise<string>((extract) => {\n      this.runIdPromiseResolver = extract;\n    });\n  }\n\n  handleChainStart = (\n    _chain: Serialized,\n    _inputs: ChainValues,\n    runId: string\n  ) => {\n    this.runIdPromiseResolver(runId);\n  };\n\n  async extract(): Promise<string> {\n    return this.runIdPromise;\n  }\n}\n\nclass SingleRunExtractor extends BaseTracer {\n  runPromiseResolver: (run: Run) => void;\n\n  runPromise: Promise<Run>;\n\n  /** The name of the callback handler. */\n  name = \"single_run_extractor\";\n\n  constructor() {\n    super();\n    this.runPromise = new Promise<Run>((extract) => {\n      this.runPromiseResolver = extract;\n    });\n  }\n\n  async persistRun(run: Run) {\n    this.runPromiseResolver(run);\n  }\n\n  async extract(): Promise<Run> {\n    return this.runPromise;\n  }\n}\n\n/**\n * Wraps an evaluator function + implements the RunEvaluator interface.\n */\nclass DynamicRunEvaluator implements RunEvaluator {\n  evaluator: RunnableLambda<DynamicRunEvaluatorParams, EvaluationResult>;\n\n  constructor(evaluator: RunEvaluatorLike) {\n    this.evaluator = new RunnableLambda({ func: evaluator });\n  }\n\n  /**\n   * Evaluates a run with an optional example and returns the evaluation result.\n   * @param run The run to evaluate.\n   * @param example The optional example to use for evaluation.\n   * @returns A promise that extracts to the evaluation result.\n   */\n  async evaluateRun(run: Run, example?: Example): Promise<EvaluationResult> {\n    const extractor = new SingleRunIdExtractor();\n    const tracer = new LangChainTracer({ projectName: \"evaluators\" });\n    const result = await this.evaluator.invoke(\n      {\n        run,\n        example,\n        input: run.inputs,\n        prediction: run.outputs,\n        reference: example?.outputs,\n      },\n      {\n        callbacks: [extractor, tracer],\n      }\n    );\n    const runId = await extractor.extract();\n    return {\n      sourceRunId: runId,\n      ...result,\n    };\n  }\n}\n\n// oxlint-disable-next-line @typescript-eslint/no-explicit-any\nfunction isLLMStringEvaluator(evaluator: any): evaluator is LLMStringEvaluator {\n  return evaluator && typeof evaluator.evaluateStrings === \"function\";\n}\n\n// oxlint-disable-next-line @typescript-eslint/no-explicit-any\ntype AnyTraceableFunction = TraceableFunction<(...any: any[]) => any>;\n\n/**\n * Internal implementation of RunTree, which uses the\n * provided callback manager instead of the internal LangSmith client.\n *\n * The goal of this class is to ensure seamless interop when intergrated\n * with other Runnables.\n */\nclass CallbackManagerRunTree extends RunTree {\n  callbackManager: CallbackManager;\n\n  activeCallbackManager: CallbackManagerForChainRun | undefined = undefined;\n\n  constructor(config: RunTreeConfig, callbackManager: CallbackManager) {\n    super(config);\n\n    this.callbackManager = callbackManager;\n  }\n\n  createChild(config: RunTreeConfig): CallbackManagerRunTree {\n    const child = new CallbackManagerRunTree(\n      {\n        ...config,\n        parent_run: this,\n        project_name: this.project_name,\n        client: this.client,\n      },\n      this.activeCallbackManager?.getChild() ?? this.callbackManager\n    );\n    this.child_runs.push(child);\n    return child;\n  }\n\n  async postRun(): Promise<void> {\n    // how it is translated in comparison to basic RunTree?\n    this.activeCallbackManager = await this.callbackManager.handleChainStart(\n      typeof this.serialized !== \"object\" &&\n        this.serialized != null &&\n        \"lc\" in this.serialized\n        ? this.serialized\n        : {\n            id: [\"langchain\", \"smith\", \"CallbackManagerRunTree\"],\n            lc: 1,\n            type: \"not_implemented\",\n          },\n      this.inputs,\n      this.id,\n      this.run_type,\n      undefined,\n      undefined,\n      this.name\n    );\n  }\n\n  async patchRun(): Promise<void> {\n    if (this.error) {\n      await this.activeCallbackManager?.handleChainError(\n        this.error,\n        this.id,\n        this.parent_run?.id,\n        undefined,\n        undefined\n      );\n    } else {\n      await this.activeCallbackManager?.handleChainEnd(\n        this.outputs ?? {},\n        this.id,\n        this.parent_run?.id,\n        undefined,\n        undefined\n      );\n    }\n  }\n}\n\nclass RunnableTraceable<RunInput, RunOutput> extends Runnable<\n  RunInput,\n  RunOutput\n> {\n  lc_serializable = false;\n\n  lc_namespace = [\"langchain_core\", \"runnables\"];\n\n  protected func: AnyTraceableFunction;\n\n  constructor(fields: { func: AnyTraceableFunction }) {\n    super(fields);\n\n    if (!isLangsmithTraceableFunction(fields.func)) {\n      throw new Error(\n        \"RunnableTraceable requires a function that is wrapped in traceable higher-order function\"\n      );\n    }\n\n    this.func = fields.func;\n  }\n\n  async invoke(input: RunInput, options?: Partial<RunnableConfig>) {\n    const [config] = this._getOptionsList(options ?? {}, 1);\n    const callbackManager = await getCallbackManagerForConfig(config);\n\n    const partialConfig =\n      \"langsmith:traceable\" in this.func\n        ? (this.func[\"langsmith:traceable\"] as RunTreeConfig)\n        : { name: \"<lambda>\" };\n\n    if (!callbackManager) throw new Error(\"CallbackManager not found\");\n    const runTree = new CallbackManagerRunTree(\n      {\n        ...partialConfig,\n        parent_run: callbackManager?._parentRunId\n          ? new RunTree({ name: \"<parent>\", id: callbackManager?._parentRunId })\n          : undefined,\n      },\n      callbackManager\n    );\n\n    if (\n      typeof input === \"object\" &&\n      input != null &&\n      Object.keys(input).length === 1\n    ) {\n      if (\"args\" in input && Array.isArray(input)) {\n        return (await this.func(runTree, ...input)) as RunOutput;\n      }\n\n      if (\n        \"input\" in input &&\n        !(\n          typeof input === \"object\" &&\n          input != null &&\n          !Array.isArray(input) &&\n          // oxlint-disable-next-line no-instanceof/no-instanceof\n          !(input instanceof Date)\n        )\n      ) {\n        try {\n          return (await this.func(runTree, input.input)) as RunOutput;\n        } catch {\n          return (await this.func(runTree, input)) as RunOutput;\n        }\n      }\n    }\n\n    return (await this.func(runTree, input)) as RunOutput;\n  }\n}\n\n/**\n * Wraps an off-the-shelf evaluator (loaded using loadEvaluator; of EvaluatorType[T])\n * and composes with a prepareData function so the user can prepare the trace and\n * dataset data for the evaluator.\n */\nclass PreparedRunEvaluator implements RunEvaluator {\n  evaluator: LLMStringEvaluator;\n\n  formatEvaluatorInputs: EvaluatorInputFormatter;\n\n  isStringEvaluator: boolean;\n\n  evaluationName: string;\n\n  constructor(\n    evaluator: LLMStringEvaluator,\n    evaluationName: string,\n    formatEvaluatorInputs: EvaluatorInputFormatter\n  ) {\n    this.evaluator = evaluator;\n    this.isStringEvaluator = typeof evaluator?.evaluateStrings === \"function\";\n    this.evaluationName = evaluationName;\n    this.formatEvaluatorInputs = formatEvaluatorInputs;\n  }\n\n  static async fromEvalConfig(\n    config: EvalConfig | keyof EvaluatorType\n  ): Promise<PreparedRunEvaluator> {\n    const evaluatorType =\n      typeof config === \"string\" ? config : config.evaluatorType;\n    const evalConfig = typeof config === \"string\" ? ({} as EvalConfig) : config;\n    const evaluator = await loadEvaluator(evaluatorType, evalConfig);\n    const feedbackKey = evalConfig?.feedbackKey ?? evaluator?.evaluationName;\n    if (!isLLMStringEvaluator(evaluator)) {\n      throw new Error(\n        `Evaluator of type ${evaluatorType} not yet supported. ` +\n          \"Please use a string evaluator, or implement your \" +\n          \"evaluation logic as a custom evaluator.\"\n      );\n    }\n    if (!feedbackKey) {\n      throw new Error(\n        `Evaluator of type ${evaluatorType} must have an evaluationName` +\n          ` or feedbackKey. Please manually provide a feedbackKey in the EvalConfig.`\n      );\n    }\n    return new PreparedRunEvaluator(\n      evaluator as LLMStringEvaluator,\n      feedbackKey,\n      evalConfig?.formatEvaluatorInputs\n    );\n  }\n\n  /**\n   * Evaluates a run with an optional example and returns the evaluation result.\n   * @param run The run to evaluate.\n   * @param example The optional example to use for evaluation.\n   * @returns A promise that extracts to the evaluation result.\n   */\n  async evaluateRun(run: Run, example?: Example): Promise<EvaluationResult> {\n    const { prediction, input, reference } = this.formatEvaluatorInputs({\n      rawInput: run.inputs,\n      rawPrediction: run.outputs,\n      rawReferenceOutput: example?.outputs,\n      run,\n    });\n    const extractor = new SingleRunIdExtractor();\n    const tracer = new LangChainTracer({ projectName: \"evaluators\" });\n    if (this.isStringEvaluator) {\n      const evalResult = await this.evaluator.evaluateStrings(\n        {\n          prediction: prediction as string,\n          reference: reference as string,\n          input: input as string,\n        },\n        {\n          callbacks: [extractor, tracer],\n        }\n      );\n      const runId = await extractor.extract();\n      return {\n        key: this.evaluationName,\n        comment: evalResult?.reasoning,\n        sourceRunId: runId,\n        ...evalResult,\n      };\n    }\n    throw new Error(\n      \"Evaluator not yet supported. \" +\n        \"Please use a string evaluator, or implement your \" +\n        \"evaluation logic as a custom evaluator.\"\n    );\n  }\n}\n\nclass LoadedEvalConfig {\n  constructor(public evaluators: (RunEvaluator | DynamicRunEvaluator)[]) {}\n\n  static async fromRunEvalConfig(\n    config: RunEvalConfig<keyof EvaluatorType>\n  ): Promise<LoadedEvalConfig> {\n    // Custom evaluators are applied \"as-is\"\n    const customEvaluators = (\n      config?.customEvaluators ?? config.evaluators?.filter(isCustomEvaluator)\n    )?.map((evaluator) => {\n      if (typeof evaluator === \"function\") {\n        return new DynamicRunEvaluator(evaluator);\n      } else {\n        return evaluator;\n      }\n    });\n\n    const offTheShelfEvaluators = await Promise.all(\n      config?.evaluators\n        ?.filter(isOffTheShelfEvaluator)\n        ?.map(\n          async (evaluator) =>\n            await PreparedRunEvaluator.fromEvalConfig(evaluator)\n        ) ?? []\n    );\n    return new LoadedEvalConfig(\n      (customEvaluators ?? []).concat(offTheShelfEvaluators ?? [])\n    );\n  }\n}\n\nexport interface RunOnDatasetParams extends Omit<\n  RunEvalConfig,\n  \"customEvaluators\"\n> {\n  /**\n   * Name of the project for logging and tracking.\n   */\n  projectName?: string;\n\n  /**\n   * Additional metadata for the project.\n   */\n  projectMetadata?: Record<string, unknown>;\n\n  /**\n   * Client instance for LangSmith service interaction.\n   */\n  client?: Client;\n\n  /**\n   * Maximum concurrency level for dataset processing.\n   */\n  maxConcurrency?: number;\n\n  /**\n   * @deprecated Pass keys directly to the RunOnDatasetParams instead\n   */\n  evaluationConfig?: RunEvalConfig;\n}\n\n/**\n * Internals expect a constructor () -> Runnable. This function wraps/coerces\n * the provided LangChain object, custom function, or factory function into\n * a constructor of a runnable.\n * @param modelOrFactory The model or factory to create a wrapped model from.\n * @returns A function that returns the wrapped model.\n * @throws Error if the modelOrFactory is invalid.\n */\nconst createWrappedModel = async (modelOrFactory: ChainOrFactory) => {\n  if (Runnable.isRunnable(modelOrFactory)) {\n    return () => modelOrFactory;\n  }\n  if (typeof modelOrFactory === \"function\") {\n    if (isLangsmithTraceableFunction(modelOrFactory)) {\n      const wrappedModel = new RunnableTraceable({ func: modelOrFactory });\n      return () => wrappedModel;\n    }\n\n    try {\n      // If it works with no arguments, assume it's a factory\n      let res = (modelOrFactory as () => Runnable)();\n      if (\n        res &&\n        typeof (res as unknown as Promise<Runnable>).then === \"function\"\n      ) {\n        res = await res;\n      }\n      return modelOrFactory as () => Runnable;\n    } catch {\n      // Otherwise, it's a custom UDF, and we'll wrap\n      // the function in a lambda\n      const wrappedModel = new RunnableLambda({ func: modelOrFactory });\n      return () => wrappedModel;\n    }\n  }\n  throw new Error(\"Invalid modelOrFactory\");\n};\n\nconst loadExamples = async ({\n  datasetName,\n  client,\n  projectName,\n}: {\n  datasetName: string;\n  client: Client;\n  projectName: string;\n  maxConcurrency: number;\n}) => {\n  const exampleIterator = client.listExamples({ datasetName });\n  const configs: RunnableConfig[] = [];\n  const runExtractors = [];\n  const examples = [];\n  for await (const example of exampleIterator) {\n    const runExtractor = new SingleRunExtractor();\n    configs.push({\n      callbacks: [\n        new LangChainTracer({ exampleId: example.id, projectName }),\n        runExtractor,\n      ],\n    });\n    examples.push(example);\n    runExtractors.push(runExtractor);\n  }\n  return {\n    configs,\n    examples,\n    runExtractors,\n  };\n};\n\nconst applyEvaluators = async ({\n  evaluation,\n  runs,\n  examples,\n  client,\n  maxConcurrency,\n}: {\n  evaluation: LoadedEvalConfig;\n  runs: Run[];\n  examples: Example[];\n  client: Client;\n  maxConcurrency: number;\n}): Promise<{\n  [key: string]: {\n    execution_time?: number;\n    run_id: string;\n    feedback: Feedback[];\n  };\n}> => {\n  // TODO: Parallelize and/or put in callbacks to speed up evals.\n  const { evaluators } = evaluation;\n  const progress = new ProgressBar({\n    total: examples.length,\n    format: \"Running Evaluators: {bar} {percentage}% | {value}/{total}\\n\",\n  });\n  const caller = new AsyncCaller({\n    maxConcurrency,\n  });\n  const requests = runs.map(\n    async (\n      run,\n      i\n    ): Promise<{\n      run_id: string;\n      execution_time?: number;\n      feedback: Feedback[];\n    }> =>\n      caller.call(async () => {\n        const evaluatorResults = await Promise.allSettled(\n          evaluators.map((evaluator) =>\n            client.evaluateRun(run, evaluator, {\n              referenceExample: examples[i],\n              loadChildRuns: false,\n            })\n          )\n        );\n        progress.increment();\n        return {\n          execution_time:\n            run?.end_time && run.start_time\n              ? new Date(run.end_time).getTime() -\n                new Date(run.start_time).getTime()\n              : undefined,\n          feedback: evaluatorResults.map((evalResult) =>\n            evalResult.status === \"fulfilled\"\n              ? evalResult.value\n              : evalResult.reason\n          ),\n          run_id: run.id,\n        };\n      })\n  );\n  const results = await Promise.all(requests);\n\n  return results.reduce(\n    (acc, result, i) => ({\n      ...acc,\n      [examples[i].id]: result,\n    }),\n    {}\n  );\n};\n\nexport type EvalResults = {\n  projectName: string;\n  results: {\n    [key: string]: {\n      execution_time?: number;\n      run_id: string;\n      feedback: Feedback[];\n    };\n  };\n};\n\nconst getExamplesInputs = (\n  examples: Example[],\n  chainOrFactory: ChainOrFactory,\n  dataType?: DataType\n) => {\n  if (dataType === \"chat\") {\n    // For some batty reason, we store the chat dataset differently.\n    // { type: \"system\", data: { content: inputs.input } },\n    // But we need to create AIMesage, SystemMessage, etc.\n    return examples.map(({ inputs }) =>\n      mapStoredMessagesToChatMessages(inputs.input)\n    );\n  }\n  // If it's a language model and ALL example inputs have a single value,\n  // then we can be friendly and flatten the inputs to a list of strings.\n  const isLanguageModel =\n    typeof chainOrFactory === \"object\" &&\n    typeof (chainOrFactory as BaseLanguageModel)._llmType === \"function\";\n  if (\n    isLanguageModel &&\n    examples.every(({ inputs }) => Object.keys(inputs).length === 1)\n  ) {\n    return examples.map(({ inputs }) => Object.values(inputs)[0]);\n  }\n  return examples.map(({ inputs }) => inputs);\n};\n\n/**\n * Evaluates a given model or chain against a specified LangSmith dataset.\n *\n * This function fetches example records from the specified dataset,\n * runs the model or chain against each example, and returns the evaluation\n * results.\n *\n * @param chainOrFactory - A model or factory/constructor function to be evaluated. It can be a\n * Runnable instance, a factory function that returns a Runnable, or a user-defined\n * function or factory.\n *\n * @param datasetName - The name of the dataset against which the evaluation will be\n * performed. This dataset should already be defined and contain the relevant data\n * for evaluation.\n *\n * @param options - (Optional) Additional parameters for the evaluation process:\n *   - `evaluators` (RunEvalType[]): Evaluators to apply to a dataset run.\n *   - `formatEvaluatorInputs` (EvaluatorInputFormatter): Convert the evaluation data into formats that can be used by the evaluator.\n *   - `projectName` (string): Name of the project for logging and tracking.\n *   - `projectMetadata` (Record<string, unknown>): Additional metadata for the project.\n *   - `client` (Client): Client instance for LangSmith service interaction.\n *   - `maxConcurrency` (number): Maximum concurrency level for dataset processing.\n *\n * @returns A promise that resolves to an `EvalResults` object. This object includes\n * detailed results of the evaluation, such as execution time, run IDs, and feedback\n * for each entry in the dataset.\n *\n * @example\n * ```typescript\n * // Example usage for evaluating a model on a dataset\n * async function evaluateModel() {\n *   const chain = /* ...create your model or chain...*\\//\n *   const datasetName = 'example-dataset';\n *   const client = new Client(/* ...config... *\\//);\n *\n *   const results = await runOnDataset(chain, datasetName, {\n *     evaluators: [/* ...evaluators... *\\//],\n *     client,\n *   });\n *\n *   console.log('Evaluation Results:', results);\n * }\n *\n * evaluateModel();\n * ```\n * In this example, `runOnDataset` is used to evaluate a language model (or a chain of models) against\n * a dataset named 'example-dataset'. The evaluation process is configured using `RunOnDatasetParams[\"evaluators\"]`, which can\n * include both standard and custom evaluators. The `Client` instance is used to interact with LangChain services.\n * The function returns the evaluation results, which can be logged or further processed as needed.\n */\n\nexport async function runOnDataset(\n  chainOrFactory: ChainOrFactory,\n  datasetName: string,\n  options?: RunOnDatasetParams\n) {\n  const {\n    projectName,\n    projectMetadata,\n    client,\n    maxConcurrency,\n  }: RunOnDatasetParams = options ?? {};\n\n  const evaluationConfig: RunEvalConfig | undefined =\n    options?.evaluationConfig ??\n    (options?.evaluators != null\n      ? {\n          evaluators: options.evaluators,\n          formatEvaluatorInputs: options.formatEvaluatorInputs,\n        }\n      : undefined);\n\n  const wrappedModel = await createWrappedModel(chainOrFactory);\n  const testClient = client ?? new Client();\n  const testProjectName = projectName ?? randomName();\n  const dataset = await testClient.readDataset({ datasetName });\n  const datasetId = dataset.id;\n  const testConcurrency = maxConcurrency ?? 5;\n  const { configs, examples, runExtractors } = await loadExamples({\n    datasetName,\n    client: testClient,\n    projectName: testProjectName,\n    maxConcurrency: testConcurrency,\n  });\n\n  await testClient.createProject({\n    projectName: testProjectName,\n    referenceDatasetId: datasetId,\n    projectExtra: { metadata: { ...projectMetadata } },\n  });\n  const wrappedRunnable: Runnable = new RunnableLambda({\n    func: wrappedModel,\n  }).withConfig({ runName: \"evaluationRun\" });\n  const runInputs = getExamplesInputs(\n    examples,\n    chainOrFactory,\n    dataset.data_type\n  );\n  const progress = new ProgressBar({\n    total: runInputs.length,\n    format: \"Predicting: {bar} {percentage}% | {value}/{total}\",\n  });\n  // TODO: Collect the runs as well.\n  await wrappedRunnable\n    .withListeners({\n      onEnd: () => progress.increment(),\n    })\n    // TODO: Insert evaluation inline for immediate feedback.\n    .batch(runInputs, configs, {\n      maxConcurrency,\n      returnExceptions: true,\n    });\n\n  progress.complete();\n  const runs: Run[] = [];\n  for (let i = 0; i < examples.length; i += 1) {\n    runs.push(await runExtractors[i].extract());\n  }\n  let evalResults: Record<\n    string,\n    { run_id: string; execution_time?: number; feedback: Feedback[] }\n  > = {};\n  if (evaluationConfig) {\n    const loadedEvalConfig =\n      await LoadedEvalConfig.fromRunEvalConfig(evaluationConfig);\n    evalResults = await applyEvaluators({\n      evaluation: loadedEvalConfig,\n      runs,\n      examples,\n      client: testClient,\n      maxConcurrency: testConcurrency,\n    });\n  }\n  const results: EvalResults = {\n    projectName: testProjectName,\n    results: evalResults ?? {},\n  };\n  return results;\n}\n\nfunction isLangsmithTraceableFunction(x: unknown): x is AnyTraceableFunction {\n  return typeof x === \"function\" && \"langsmith:traceable\" in x;\n}\n"],"mappings":";;;;;;;;;;;AAsDA,IAAM,uBAAN,MAA2B;CACzB;CAEA;CAEA,cAAc;AACZ,OAAK,eAAe,IAAI,SAAiB,YAAY;AACnD,QAAK,uBAAuB;IAC5B;;CAGJ,oBACE,QACA,SACA,UACG;AACH,OAAK,qBAAqB,MAAM;;CAGlC,MAAM,UAA2B;AAC/B,SAAO,KAAK;;;AAIhB,IAAM,qBAAN,cAAiC,WAAW;CAC1C;CAEA;;CAGA,OAAO;CAEP,cAAc;AACZ,SAAO;AACP,OAAK,aAAa,IAAI,SAAc,YAAY;AAC9C,QAAK,qBAAqB;IAC1B;;CAGJ,MAAM,WAAW,KAAU;AACzB,OAAK,mBAAmB,IAAI;;CAG9B,MAAM,UAAwB;AAC5B,SAAO,KAAK;;;;;;AAOhB,IAAM,sBAAN,MAAkD;CAChD;CAEA,YAAY,WAA6B;AACvC,OAAK,YAAY,IAAI,eAAe,EAAE,MAAM,WAAW,CAAC;;;;;;;;CAS1D,MAAM,YAAY,KAAU,SAA8C;EACxE,MAAM,YAAY,IAAI,sBAAsB;EAC5C,MAAM,SAAS,IAAI,gBAAgB,EAAE,aAAa,cAAc,CAAC;EACjE,MAAM,SAAS,MAAM,KAAK,UAAU,OAClC;GACE;GACA;GACA,OAAO,IAAI;GACX,YAAY,IAAI;GAChB,WAAW,SAAS;GACrB,EACD,EACE,WAAW,CAAC,WAAW,OAAO,EAC/B,CACF;AAED,SAAO;GACL,aAAa,MAFK,UAAU,SAAS;GAGrC,GAAG;GACJ;;;AAKL,SAAS,qBAAqB,WAAiD;AAC7E,QAAO,aAAa,OAAO,UAAU,oBAAoB;;;;;;;;;AAa3D,IAAM,yBAAN,MAAM,+BAA+B,QAAQ;CAC3C;CAEA,wBAAgE,KAAA;CAEhE,YAAY,QAAuB,iBAAkC;AACnE,QAAM,OAAO;AAEb,OAAK,kBAAkB;;CAGzB,YAAY,QAA+C;EACzD,MAAM,QAAQ,IAAI,uBAChB;GACE,GAAG;GACH,YAAY;GACZ,cAAc,KAAK;GACnB,QAAQ,KAAK;GACd,EACD,KAAK,uBAAuB,UAAU,IAAI,KAAK,gBAChD;AACD,OAAK,WAAW,KAAK,MAAM;AAC3B,SAAO;;CAGT,MAAM,UAAyB;AAE7B,OAAK,wBAAwB,MAAM,KAAK,gBAAgB,iBACtD,OAAO,KAAK,eAAe,YACzB,KAAK,cAAc,QACnB,QAAQ,KAAK,aACX,KAAK,aACL;GACE,IAAI;IAAC;IAAa;IAAS;IAAyB;GACpD,IAAI;GACJ,MAAM;GACP,EACL,KAAK,QACL,KAAK,IACL,KAAK,UACL,KAAA,GACA,KAAA,GACA,KAAK,KACN;;CAGH,MAAM,WAA0B;AAC9B,MAAI,KAAK,MACP,OAAM,KAAK,uBAAuB,iBAChC,KAAK,OACL,KAAK,IACL,KAAK,YAAY,IACjB,KAAA,GACA,KAAA,EACD;MAED,OAAM,KAAK,uBAAuB,eAChC,KAAK,WAAW,EAAE,EAClB,KAAK,IACL,KAAK,YAAY,IACjB,KAAA,GACA,KAAA,EACD;;;AAKP,IAAM,oBAAN,cAAqD,SAGnD;CACA,kBAAkB;CAElB,eAAe,CAAC,kBAAkB,YAAY;CAE9C;CAEA,YAAY,QAAwC;AAClD,QAAM,OAAO;AAEb,MAAI,CAAC,6BAA6B,OAAO,KAAK,CAC5C,OAAM,IAAI,MACR,2FACD;AAGH,OAAK,OAAO,OAAO;;CAGrB,MAAM,OAAO,OAAiB,SAAmC;EAC/D,MAAM,CAAC,UAAU,KAAK,gBAAgB,WAAW,EAAE,EAAE,EAAE;EACvD,MAAM,kBAAkB,MAAM,4BAA4B,OAAO;EAEjE,MAAM,gBACJ,yBAAyB,KAAK,OACzB,KAAK,KAAK,yBACX,EAAE,MAAM,YAAY;AAE1B,MAAI,CAAC,gBAAiB,OAAM,IAAI,MAAM,4BAA4B;EAClE,MAAM,UAAU,IAAI,uBAClB;GACE,GAAG;GACH,YAAY,iBAAiB,eACzB,IAAI,QAAQ;IAAE,MAAM;IAAY,IAAI,iBAAiB;IAAc,CAAC,GACpE,KAAA;GACL,EACD,gBACD;AAED,MACE,OAAO,UAAU,YACjB,SAAS,QACT,OAAO,KAAK,MAAM,CAAC,WAAW,GAC9B;AACA,OAAI,UAAU,SAAS,MAAM,QAAQ,MAAM,CACzC,QAAQ,MAAM,KAAK,KAAK,SAAS,GAAG,MAAM;AAG5C,OACE,WAAW,SACX,EACE,OAAO,UAAU,YACjB,SAAS,QACT,CAAC,MAAM,QAAQ,MAAM,IAErB,EAAE,iBAAiB,OAGrB,KAAI;AACF,WAAQ,MAAM,KAAK,KAAK,SAAS,MAAM,MAAM;WACvC;AACN,WAAQ,MAAM,KAAK,KAAK,SAAS,MAAM;;;AAK7C,SAAQ,MAAM,KAAK,KAAK,SAAS,MAAM;;;;;;;;AAS3C,IAAM,uBAAN,MAAM,qBAA6C;CACjD;CAEA;CAEA;CAEA;CAEA,YACE,WACA,gBACA,uBACA;AACA,OAAK,YAAY;AACjB,OAAK,oBAAoB,OAAO,WAAW,oBAAoB;AAC/D,OAAK,iBAAiB;AACtB,OAAK,wBAAwB;;CAG/B,aAAa,eACX,QAC+B;EAC/B,MAAM,gBACJ,OAAO,WAAW,WAAW,SAAS,OAAO;EAC/C,MAAM,aAAa,OAAO,WAAW,WAAY,EAAE,GAAkB;EACrE,MAAM,YAAY,MAAM,cAAc,eAAe,WAAW;EAChE,MAAM,cAAc,YAAY,eAAe,WAAW;AAC1D,MAAI,CAAC,qBAAqB,UAAU,CAClC,OAAM,IAAI,MACR,qBAAqB,cAAc,8GAGpC;AAEH,MAAI,CAAC,YACH,OAAM,IAAI,MACR,qBAAqB,cAAc,uGAEpC;AAEH,SAAO,IAAI,qBACT,WACA,aACA,YAAY,sBACb;;;;;;;;CASH,MAAM,YAAY,KAAU,SAA8C;EACxE,MAAM,EAAE,YAAY,OAAO,cAAc,KAAK,sBAAsB;GAClE,UAAU,IAAI;GACd,eAAe,IAAI;GACnB,oBAAoB,SAAS;GAC7B;GACD,CAAC;EACF,MAAM,YAAY,IAAI,sBAAsB;EAC5C,MAAM,SAAS,IAAI,gBAAgB,EAAE,aAAa,cAAc,CAAC;AACjE,MAAI,KAAK,mBAAmB;GAC1B,MAAM,aAAa,MAAM,KAAK,UAAU,gBACtC;IACc;IACD;IACJ;IACR,EACD,EACE,WAAW,CAAC,WAAW,OAAO,EAC/B,CACF;GACD,MAAM,QAAQ,MAAM,UAAU,SAAS;AACvC,UAAO;IACL,KAAK,KAAK;IACV,SAAS,YAAY;IACrB,aAAa;IACb,GAAG;IACJ;;AAEH,QAAM,IAAI,MACR,wHAGD;;;AAIL,IAAM,mBAAN,MAAM,iBAAiB;CACrB,YAAY,YAA2D;AAApD,OAAA,aAAA;;CAEnB,aAAa,kBACX,QAC2B;EAE3B,MAAM,oBACJ,QAAQ,oBAAoB,OAAO,YAAY,OAAO,kBAAkB,GACvE,KAAK,cAAc;AACpB,OAAI,OAAO,cAAc,WACvB,QAAO,IAAI,oBAAoB,UAAU;OAEzC,QAAO;IAET;EAEF,MAAM,wBAAwB,MAAM,QAAQ,IAC1C,QAAQ,YACJ,OAAO,uBAAuB,EAC9B,IACA,OAAO,cACL,MAAM,qBAAqB,eAAe,UAAU,CACvD,IAAI,EAAE,CACV;AACD,SAAO,IAAI,kBACR,oBAAoB,EAAE,EAAE,OAAO,yBAAyB,EAAE,CAAC,CAC7D;;;;;;;;;;;AA0CL,MAAM,qBAAqB,OAAO,mBAAmC;AACnE,KAAI,SAAS,WAAW,eAAe,CACrC,cAAa;AAEf,KAAI,OAAO,mBAAmB,YAAY;AACxC,MAAI,6BAA6B,eAAe,EAAE;GAChD,MAAM,eAAe,IAAI,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AACpE,gBAAa;;AAGf,MAAI;GAEF,IAAI,MAAO,gBAAmC;AAC9C,OACE,OACA,OAAQ,IAAqC,SAAS,WAEtD,OAAM,MAAM;AAEd,UAAO;UACD;GAGN,MAAM,eAAe,IAAI,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACjE,gBAAa;;;AAGjB,OAAM,IAAI,MAAM,yBAAyB;;AAG3C,MAAM,eAAe,OAAO,EAC1B,aACA,QACA,kBAMI;CACJ,MAAM,kBAAkB,OAAO,aAAa,EAAE,aAAa,CAAC;CAC5D,MAAM,UAA4B,EAAE;CACpC,MAAM,gBAAgB,EAAE;CACxB,MAAM,WAAW,EAAE;AACnB,YAAW,MAAM,WAAW,iBAAiB;EAC3C,MAAM,eAAe,IAAI,oBAAoB;AAC7C,UAAQ,KAAK,EACX,WAAW,CACT,IAAI,gBAAgB;GAAE,WAAW,QAAQ;GAAI;GAAa,CAAC,EAC3D,aACD,EACF,CAAC;AACF,WAAS,KAAK,QAAQ;AACtB,gBAAc,KAAK,aAAa;;AAElC,QAAO;EACL;EACA;EACA;EACD;;AAGH,MAAM,kBAAkB,OAAO,EAC7B,YACA,MACA,UACA,QACA,qBAaI;CAEJ,MAAM,EAAE,eAAe;CACvB,MAAM,WAAW,IAAI,YAAY;EAC/B,OAAO,SAAS;EAChB,QAAQ;EACT,CAAC;CACF,MAAM,SAAS,IAAI,YAAY,EAC7B,gBACD,CAAC;CACF,MAAM,WAAW,KAAK,IACpB,OACE,KACA,MAMA,OAAO,KAAK,YAAY;EACtB,MAAM,mBAAmB,MAAM,QAAQ,WACrC,WAAW,KAAK,cACd,OAAO,YAAY,KAAK,WAAW;GACjC,kBAAkB,SAAS;GAC3B,eAAe;GAChB,CAAC,CACH,CACF;AACD,WAAS,WAAW;AACpB,SAAO;GACL,gBACE,KAAK,YAAY,IAAI,aACjB,IAAI,KAAK,IAAI,SAAS,CAAC,SAAS,GAChC,IAAI,KAAK,IAAI,WAAW,CAAC,SAAS,GAClC,KAAA;GACN,UAAU,iBAAiB,KAAK,eAC9B,WAAW,WAAW,cAClB,WAAW,QACX,WAAW,OAChB;GACD,QAAQ,IAAI;GACb;GACD,CACL;AAGD,SAAO,MAFe,QAAQ,IAAI,SAAS,EAE5B,QACZ,KAAK,QAAQ,OAAO;EACnB,GAAG;GACF,SAAS,GAAG,KAAK;EACnB,GACD,EAAE,CACH;;AAcH,MAAM,qBACJ,UACA,gBACA,aACG;AACH,KAAI,aAAa,OAIf,QAAO,SAAS,KAAK,EAAE,aACrB,gCAAgC,OAAO,MAAM,CAC9C;AAOH,KAFE,OAAO,mBAAmB,YAC1B,OAAQ,eAAqC,aAAa,cAG1D,SAAS,OAAO,EAAE,aAAa,OAAO,KAAK,OAAO,CAAC,WAAW,EAAE,CAEhE,QAAO,SAAS,KAAK,EAAE,aAAa,OAAO,OAAO,OAAO,CAAC,GAAG;AAE/D,QAAO,SAAS,KAAK,EAAE,aAAa,OAAO;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAsD7C,eAAsB,aACpB,gBACA,aACA,SACA;CACA,MAAM,EACJ,aACA,iBACA,QACA,mBACsB,WAAW,EAAE;CAErC,MAAM,mBACJ,SAAS,qBACR,SAAS,cAAc,OACpB;EACE,YAAY,QAAQ;EACpB,uBAAuB,QAAQ;EAChC,GACD,KAAA;CAEN,MAAM,eAAe,MAAM,mBAAmB,eAAe;CAC7D,MAAM,aAAa,UAAU,IAAI,QAAQ;CACzC,MAAM,kBAAkB,eAAe,YAAY;CACnD,MAAM,UAAU,MAAM,WAAW,YAAY,EAAE,aAAa,CAAC;CAC7D,MAAM,YAAY,QAAQ;CAC1B,MAAM,kBAAkB,kBAAkB;CAC1C,MAAM,EAAE,SAAS,UAAU,kBAAkB,MAAM,aAAa;EAC9D;EACA,QAAQ;EACR,aAAa;EACb,gBAAgB;EACjB,CAAC;AAEF,OAAM,WAAW,cAAc;EAC7B,aAAa;EACb,oBAAoB;EACpB,cAAc,EAAE,UAAU,EAAE,GAAG,iBAAiB,EAAE;EACnD,CAAC;CACF,MAAM,kBAA4B,IAAI,eAAe,EACnD,MAAM,cACP,CAAC,CAAC,WAAW,EAAE,SAAS,iBAAiB,CAAC;CAC3C,MAAM,YAAY,kBAChB,UACA,gBACA,QAAQ,UACT;CACD,MAAM,WAAW,IAAI,YAAY;EAC/B,OAAO,UAAU;EACjB,QAAQ;EACT,CAAC;AAEF,OAAM,gBACH,cAAc,EACb,aAAa,SAAS,WAAW,EAClC,CAAC,CAED,MAAM,WAAW,SAAS;EACzB;EACA,kBAAkB;EACnB,CAAC;AAEJ,UAAS,UAAU;CACnB,MAAM,OAAc,EAAE;AACtB,MAAK,IAAI,IAAI,GAAG,IAAI,SAAS,QAAQ,KAAK,EACxC,MAAK,KAAK,MAAM,cAAc,GAAG,SAAS,CAAC;CAE7C,IAAI,cAGA,EAAE;AACN,KAAI,iBAGF,eAAc,MAAM,gBAAgB;EAClC,YAAY,MAFN,iBAAiB,kBAAkB,iBAAiB;EAG1D;EACA;EACA,QAAQ;EACR,gBAAgB;EACjB,CAAC;AAMJ,QAAO;EAHL,aAAa;EACb,SAAS,eAAe,EAAE;EAEd;;AAGhB,SAAS,6BAA6B,GAAuC;AAC3E,QAAO,OAAO,MAAM,cAAc,yBAAyB"}
+{"version":3,"file":"runner_utils.js","names":[],"sources":["../../src/smith/runner_utils.ts"],"sourcesContent":["import { BaseLanguageModel } from \"@langchain/core/language_models/base\";\nimport { Serialized } from \"@langchain/core/load/serializable\";\nimport { mapStoredMessagesToChatMessages } from \"@langchain/core/messages\";\nimport {\n  Runnable,\n  RunnableConfig,\n  RunnableLambda,\n  getCallbackManagerForConfig,\n} from \"@langchain/core/runnables\";\nimport { LangChainTracer } from \"@langchain/core/tracers/tracer_langchain\";\nimport { BaseTracer } from \"@langchain/core/tracers/base\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { AsyncCaller } from \"@langchain/core/utils/async_caller\";\nimport type {\n  CallbackManager,\n  CallbackManagerForChainRun,\n} from \"@langchain/core/callbacks/manager\";\nimport {\n  Client,\n  Example,\n  Feedback,\n  Run,\n  RunTree,\n  RunTreeConfig,\n} from \"langsmith\";\nimport { EvaluationResult, RunEvaluator } from \"langsmith/evaluation\";\nimport { DataType } from \"langsmith/schemas\";\nimport type { TraceableFunction } from \"langsmith/singletons/traceable\";\nimport { LLMStringEvaluator } from \"../evaluation/base.js\";\nimport { loadEvaluator } from \"../evaluation/loader.js\";\nimport { EvaluatorType } from \"../evaluation/types.js\";\nimport {\n  isOffTheShelfEvaluator,\n  type DynamicRunEvaluatorParams,\n  type EvalConfig,\n  type EvaluatorInputFormatter,\n  type RunEvalConfig,\n  type RunEvaluatorLike,\n  isCustomEvaluator,\n} from \"./config.js\";\nimport { randomName } from \"./name_generation.js\";\nimport { ProgressBar } from \"./progress.js\";\n\nexport type ChainOrFactory =\n  | Runnable\n  | (() => Runnable)\n  | AnyTraceableFunction\n  // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n  | ((obj: any) => any)\n  // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n  | ((obj: any) => Promise<any>)\n  | (() => (obj: unknown) => unknown)\n  | (() => (obj: unknown) => Promise<unknown>);\n\nclass SingleRunIdExtractor {\n  runIdPromiseResolver: (runId: string) => void;\n\n  runIdPromise: Promise<string>;\n\n  constructor() {\n    this.runIdPromise = new Promise<string>((extract) => {\n      this.runIdPromiseResolver = extract;\n    });\n  }\n\n  handleChainStart = (\n    _chain: Serialized,\n    _inputs: ChainValues,\n    runId: string\n  ) => {\n    this.runIdPromiseResolver(runId);\n  };\n\n  async extract(): Promise<string> {\n    return this.runIdPromise;\n  }\n}\n\nclass SingleRunExtractor extends BaseTracer {\n  runPromiseResolver: (run: Run) => void;\n\n  runPromise: Promise<Run>;\n\n  /** The name of the callback handler. */\n  name = \"single_run_extractor\";\n\n  constructor() {\n    super();\n    this.runPromise = new Promise<Run>((extract) => {\n      this.runPromiseResolver = extract;\n    });\n  }\n\n  async persistRun(run: Run) {\n    this.runPromiseResolver(run);\n  }\n\n  async extract(): Promise<Run> {\n    return this.runPromise;\n  }\n}\n\n/**\n * Wraps an evaluator function + implements the RunEvaluator interface.\n */\nclass DynamicRunEvaluator implements RunEvaluator {\n  evaluator: RunnableLambda<DynamicRunEvaluatorParams, EvaluationResult>;\n\n  constructor(evaluator: RunEvaluatorLike) {\n    this.evaluator = new RunnableLambda({ func: evaluator });\n  }\n\n  /**\n   * Evaluates a run with an optional example and returns the evaluation result.\n   * @param run The run to evaluate.\n   * @param example The optional example to use for evaluation.\n   * @returns A promise that extracts to the evaluation result.\n   */\n  async evaluateRun(run: Run, example?: Example): Promise<EvaluationResult> {\n    const extractor = new SingleRunIdExtractor();\n    const tracer = new LangChainTracer({ projectName: \"evaluators\" });\n    const result = await this.evaluator.invoke(\n      {\n        run,\n        example,\n        input: run.inputs,\n        prediction: run.outputs,\n        reference: example?.outputs,\n      },\n      {\n        callbacks: [extractor, tracer],\n      }\n    );\n    const runId = await extractor.extract();\n    return {\n      sourceRunId: runId,\n      ...result,\n    };\n  }\n}\n\n// oxlint-disable-next-line @typescript-eslint/no-explicit-any\nfunction isLLMStringEvaluator(evaluator: any): evaluator is LLMStringEvaluator {\n  return evaluator && typeof evaluator.evaluateStrings === \"function\";\n}\n\n// oxlint-disable-next-line @typescript-eslint/no-explicit-any\ntype AnyTraceableFunction = TraceableFunction<(...any: any[]) => any>;\n\n/**\n * Internal implementation of RunTree, which uses the\n * provided callback manager instead of the internal LangSmith client.\n *\n * The goal of this class is to ensure seamless interop when intergrated\n * with other Runnables.\n */\nclass CallbackManagerRunTree extends RunTree {\n  callbackManager: CallbackManager;\n\n  activeCallbackManager: CallbackManagerForChainRun | undefined = undefined;\n\n  constructor(config: RunTreeConfig, callbackManager: CallbackManager) {\n    super(config);\n\n    this.callbackManager = callbackManager;\n  }\n\n  createChild(config: RunTreeConfig): CallbackManagerRunTree {\n    const child = new CallbackManagerRunTree(\n      {\n        ...config,\n        parent_run: this,\n        project_name: this.project_name,\n        client: this.client,\n      },\n      this.activeCallbackManager?.getChild() ?? this.callbackManager\n    );\n    this.child_runs.push(child);\n    return child;\n  }\n\n  async postRun(): Promise<void> {\n    // how it is translated in comparison to basic RunTree?\n    this.activeCallbackManager = await this.callbackManager.handleChainStart(\n      typeof this.serialized !== \"object\" &&\n        this.serialized != null &&\n        \"lc\" in this.serialized\n        ? this.serialized\n        : {\n            id: [\"langchain\", \"smith\", \"CallbackManagerRunTree\"],\n            lc: 1,\n            type: \"not_implemented\",\n          },\n      this.inputs,\n      this.id,\n      this.run_type,\n      undefined,\n      undefined,\n      this.name\n    );\n  }\n\n  async patchRun(): Promise<void> {\n    if (this.error) {\n      await this.activeCallbackManager?.handleChainError(\n        this.error,\n        this.id,\n        this.parent_run?.id,\n        undefined,\n        undefined\n      );\n    } else {\n      await this.activeCallbackManager?.handleChainEnd(\n        this.outputs ?? {},\n        this.id,\n        this.parent_run?.id,\n        undefined,\n        undefined\n      );\n    }\n  }\n}\n\nclass RunnableTraceable<RunInput, RunOutput> extends Runnable<\n  RunInput,\n  RunOutput\n> {\n  lc_serializable = false;\n\n  lc_namespace = [\"langchain_core\", \"runnables\"];\n\n  protected func: AnyTraceableFunction;\n\n  constructor(fields: { func: AnyTraceableFunction }) {\n    super(fields);\n\n    if (!isLangsmithTraceableFunction(fields.func)) {\n      throw new Error(\n        \"RunnableTraceable requires a function that is wrapped in traceable higher-order function\"\n      );\n    }\n\n    this.func = fields.func;\n  }\n\n  async invoke(input: RunInput, options?: Partial<RunnableConfig>) {\n    const [config] = this._getOptionsList(options ?? {}, 1);\n    const callbackManager = await getCallbackManagerForConfig(config);\n\n    const partialConfig =\n      \"langsmith:traceable\" in this.func\n        ? (this.func[\"langsmith:traceable\"] as RunTreeConfig)\n        : { name: \"<lambda>\" };\n\n    if (!callbackManager) throw new Error(\"CallbackManager not found\");\n    const runTree = new CallbackManagerRunTree(\n      {\n        ...partialConfig,\n        parent_run: callbackManager?._parentRunId\n          ? new RunTree({ name: \"<parent>\", id: callbackManager?._parentRunId })\n          : undefined,\n      },\n      callbackManager\n    );\n\n    if (\n      typeof input === \"object\" &&\n      input != null &&\n      Object.keys(input).length === 1\n    ) {\n      if (\"args\" in input && Array.isArray(input)) {\n        return (await this.func(runTree, ...input)) as RunOutput;\n      }\n\n      if (\n        \"input\" in input &&\n        !(\n          typeof input === \"object\" &&\n          input != null &&\n          !Array.isArray(input) &&\n          // oxlint-disable-next-line no-instanceof/no-instanceof\n          !(input instanceof Date)\n        )\n      ) {\n        try {\n          return (await this.func(runTree, input.input)) as RunOutput;\n        } catch {\n          return (await this.func(runTree, input)) as RunOutput;\n        }\n      }\n    }\n\n    return (await this.func(runTree, input)) as RunOutput;\n  }\n}\n\n/**\n * Wraps an off-the-shelf evaluator (loaded using loadEvaluator; of EvaluatorType[T])\n * and composes with a prepareData function so the user can prepare the trace and\n * dataset data for the evaluator.\n */\nclass PreparedRunEvaluator implements RunEvaluator {\n  evaluator: LLMStringEvaluator;\n\n  formatEvaluatorInputs: EvaluatorInputFormatter;\n\n  isStringEvaluator: boolean;\n\n  evaluationName: string;\n\n  constructor(\n    evaluator: LLMStringEvaluator,\n    evaluationName: string,\n    formatEvaluatorInputs: EvaluatorInputFormatter\n  ) {\n    this.evaluator = evaluator;\n    this.isStringEvaluator = typeof evaluator?.evaluateStrings === \"function\";\n    this.evaluationName = evaluationName;\n    this.formatEvaluatorInputs = formatEvaluatorInputs;\n  }\n\n  static async fromEvalConfig(\n    config: EvalConfig | keyof EvaluatorType\n  ): Promise<PreparedRunEvaluator> {\n    const evaluatorType =\n      typeof config === \"string\" ? config : config.evaluatorType;\n    const evalConfig = typeof config === \"string\" ? ({} as EvalConfig) : config;\n    const evaluator = await loadEvaluator(evaluatorType, evalConfig);\n    const feedbackKey = evalConfig?.feedbackKey ?? evaluator?.evaluationName;\n    if (!isLLMStringEvaluator(evaluator)) {\n      throw new Error(\n        `Evaluator of type ${evaluatorType} not yet supported. ` +\n          \"Please use a string evaluator, or implement your \" +\n          \"evaluation logic as a custom evaluator.\"\n      );\n    }\n    if (!feedbackKey) {\n      throw new Error(\n        `Evaluator of type ${evaluatorType} must have an evaluationName` +\n          ` or feedbackKey. Please manually provide a feedbackKey in the EvalConfig.`\n      );\n    }\n    return new PreparedRunEvaluator(\n      evaluator as LLMStringEvaluator,\n      feedbackKey,\n      evalConfig?.formatEvaluatorInputs\n    );\n  }\n\n  /**\n   * Evaluates a run with an optional example and returns the evaluation result.\n   * @param run The run to evaluate.\n   * @param example The optional example to use for evaluation.\n   * @returns A promise that extracts to the evaluation result.\n   */\n  async evaluateRun(run: Run, example?: Example): Promise<EvaluationResult> {\n    const { prediction, input, reference } = this.formatEvaluatorInputs({\n      rawInput: run.inputs,\n      rawPrediction: run.outputs,\n      rawReferenceOutput: example?.outputs,\n      run,\n    });\n    const extractor = new SingleRunIdExtractor();\n    const tracer = new LangChainTracer({ projectName: \"evaluators\" });\n    if (this.isStringEvaluator) {\n      const evalResult = await this.evaluator.evaluateStrings(\n        {\n          prediction: prediction as string,\n          reference: reference as string,\n          input: input as string,\n        },\n        {\n          callbacks: [extractor, tracer],\n        }\n      );\n      const runId = await extractor.extract();\n      return {\n        key: this.evaluationName,\n        comment: evalResult?.reasoning,\n        sourceRunId: runId,\n        ...evalResult,\n      };\n    }\n    throw new Error(\n      \"Evaluator not yet supported. \" +\n        \"Please use a string evaluator, or implement your \" +\n        \"evaluation logic as a custom evaluator.\"\n    );\n  }\n}\n\nclass LoadedEvalConfig {\n  constructor(public evaluators: (RunEvaluator | DynamicRunEvaluator)[]) {}\n\n  static async fromRunEvalConfig(\n    config: RunEvalConfig<keyof EvaluatorType>\n  ): Promise<LoadedEvalConfig> {\n    // Custom evaluators are applied \"as-is\"\n    const customEvaluators = (\n      config?.customEvaluators ?? config.evaluators?.filter(isCustomEvaluator)\n    )?.map((evaluator) => {\n      if (typeof evaluator === \"function\") {\n        return new DynamicRunEvaluator(evaluator);\n      } else {\n        return evaluator;\n      }\n    });\n\n    const offTheShelfEvaluators = await Promise.all(\n      config?.evaluators\n        ?.filter(isOffTheShelfEvaluator)\n        ?.map(\n          async (evaluator) =>\n            await PreparedRunEvaluator.fromEvalConfig(evaluator)\n        ) ?? []\n    );\n    return new LoadedEvalConfig(\n      (customEvaluators ?? []).concat(offTheShelfEvaluators ?? [])\n    );\n  }\n}\n\nexport interface RunOnDatasetParams extends Omit<\n  RunEvalConfig,\n  \"customEvaluators\"\n> {\n  /**\n   * Name of the project for logging and tracking.\n   */\n  projectName?: string;\n\n  /**\n   * Additional metadata for the project.\n   */\n  projectMetadata?: Record<string, unknown>;\n\n  /**\n   * Client instance for LangSmith service interaction.\n   */\n  client?: Client;\n\n  /**\n   * Maximum concurrency level for dataset processing.\n   */\n  maxConcurrency?: number;\n\n  /**\n   * @deprecated Pass keys directly to the RunOnDatasetParams instead\n   */\n  evaluationConfig?: RunEvalConfig;\n}\n\n/**\n * Internals expect a constructor () -> Runnable. This function wraps/coerces\n * the provided LangChain object, custom function, or factory function into\n * a constructor of a runnable.\n * @param modelOrFactory The model or factory to create a wrapped model from.\n * @returns A function that returns the wrapped model.\n * @throws Error if the modelOrFactory is invalid.\n */\nconst createWrappedModel = async (modelOrFactory: ChainOrFactory) => {\n  if (Runnable.isRunnable(modelOrFactory)) {\n    return () => modelOrFactory;\n  }\n  if (typeof modelOrFactory === \"function\") {\n    if (isLangsmithTraceableFunction(modelOrFactory)) {\n      const wrappedModel = new RunnableTraceable({ func: modelOrFactory });\n      return () => wrappedModel;\n    }\n\n    try {\n      // If it works with no arguments, assume it's a factory\n      let res = (modelOrFactory as () => Runnable)();\n      if (\n        res &&\n        typeof (res as unknown as Promise<Runnable>).then === \"function\"\n      ) {\n        res = await res;\n      }\n      return modelOrFactory as () => Runnable;\n    } catch {\n      // Otherwise, it's a custom UDF, and we'll wrap\n      // the function in a lambda\n      const wrappedModel = new RunnableLambda({ func: modelOrFactory });\n      return () => wrappedModel;\n    }\n  }\n  throw new Error(\"Invalid modelOrFactory\");\n};\n\nconst loadExamples = async ({\n  datasetName,\n  client,\n  projectName,\n}: {\n  datasetName: string;\n  client: Client;\n  projectName: string;\n  maxConcurrency: number;\n}) => {\n  const exampleIterator = client.listExamples({ datasetName });\n  const configs: RunnableConfig[] = [];\n  const runExtractors = [];\n  const examples = [];\n  for await (const example of exampleIterator) {\n    const runExtractor = new SingleRunExtractor();\n    configs.push({\n      callbacks: [\n        new LangChainTracer({ exampleId: example.id, projectName }),\n        runExtractor,\n      ],\n    });\n    examples.push(example);\n    runExtractors.push(runExtractor);\n  }\n  return {\n    configs,\n    examples,\n    runExtractors,\n  };\n};\n\nconst applyEvaluators = async ({\n  evaluation,\n  runs,\n  examples,\n  client,\n  maxConcurrency,\n}: {\n  evaluation: LoadedEvalConfig;\n  runs: Run[];\n  examples: Example[];\n  client: Client;\n  maxConcurrency: number;\n}): Promise<{\n  [key: string]: {\n    execution_time?: number;\n    run_id: string;\n    feedback: Feedback[];\n  };\n}> => {\n  // TODO: Parallelize and/or put in callbacks to speed up evals.\n  const { evaluators } = evaluation;\n  const progress = new ProgressBar({\n    total: examples.length,\n    format: \"Running Evaluators: {bar} {percentage}% | {value}/{total}\\n\",\n  });\n  const caller = new AsyncCaller({\n    maxConcurrency,\n  });\n  const requests = runs.map(\n    async (\n      run,\n      i\n    ): Promise<{\n      run_id: string;\n      execution_time?: number;\n      feedback: Feedback[];\n    }> =>\n      caller.call(async () => {\n        const evaluatorResults = await Promise.allSettled(\n          evaluators.map((evaluator) =>\n            client.evaluateRun(run, evaluator, {\n              referenceExample: examples[i],\n              loadChildRuns: false,\n            })\n          )\n        );\n        progress.increment();\n        return {\n          execution_time:\n            run?.end_time && run.start_time\n              ? new Date(run.end_time).getTime() -\n                new Date(run.start_time).getTime()\n              : undefined,\n          feedback: evaluatorResults.map((evalResult) =>\n            evalResult.status === \"fulfilled\"\n              ? evalResult.value\n              : evalResult.reason\n          ),\n          run_id: run.id,\n        };\n      })\n  );\n  const results = await Promise.all(requests);\n\n  return results.reduce(\n    (acc, result, i) => ({\n      ...acc,\n      [examples[i].id]: result,\n    }),\n    {}\n  );\n};\n\nexport type EvalResults = {\n  projectName: string;\n  results: {\n    [key: string]: {\n      execution_time?: number;\n      run_id: string;\n      feedback: Feedback[];\n    };\n  };\n};\n\nconst getExamplesInputs = (\n  examples: Example[],\n  chainOrFactory: ChainOrFactory,\n  dataType?: DataType\n) => {\n  if (dataType === \"chat\") {\n    // For some batty reason, we store the chat dataset differently.\n    // { type: \"system\", data: { content: inputs.input } },\n    // But we need to create AIMesage, SystemMessage, etc.\n    return examples.map(({ inputs }) =>\n      mapStoredMessagesToChatMessages(inputs.input)\n    );\n  }\n  // If it's a language model and ALL example inputs have a single value,\n  // then we can be friendly and flatten the inputs to a list of strings.\n  const isLanguageModel =\n    typeof chainOrFactory === \"object\" &&\n    typeof (chainOrFactory as BaseLanguageModel)._llmType === \"function\";\n  if (\n    isLanguageModel &&\n    examples.every(({ inputs }) => Object.keys(inputs).length === 1)\n  ) {\n    return examples.map(({ inputs }) => Object.values(inputs)[0]);\n  }\n  return examples.map(({ inputs }) => inputs);\n};\n\n/**\n * Evaluates a given model or chain against a specified LangSmith dataset.\n *\n * This function fetches example records from the specified dataset,\n * runs the model or chain against each example, and returns the evaluation\n * results.\n *\n * @param chainOrFactory - A model or factory/constructor function to be evaluated. It can be a\n * Runnable instance, a factory function that returns a Runnable, or a user-defined\n * function or factory.\n *\n * @param datasetName - The name of the dataset against which the evaluation will be\n * performed. This dataset should already be defined and contain the relevant data\n * for evaluation.\n *\n * @param options - (Optional) Additional parameters for the evaluation process:\n *   - `evaluators` (RunEvalType[]): Evaluators to apply to a dataset run.\n *   - `formatEvaluatorInputs` (EvaluatorInputFormatter): Convert the evaluation data into formats that can be used by the evaluator.\n *   - `projectName` (string): Name of the project for logging and tracking.\n *   - `projectMetadata` (Record<string, unknown>): Additional metadata for the project.\n *   - `client` (Client): Client instance for LangSmith service interaction.\n *   - `maxConcurrency` (number): Maximum concurrency level for dataset processing.\n *\n * @returns A promise that resolves to an `EvalResults` object. This object includes\n * detailed results of the evaluation, such as execution time, run IDs, and feedback\n * for each entry in the dataset.\n *\n * @example\n * ```typescript\n * // Example usage for evaluating a model on a dataset\n * async function evaluateModel() {\n *   const chain = /* ...create your model or chain...*\\//\n *   const datasetName = 'example-dataset';\n *   const client = new Client(/* ...config... *\\//);\n *\n *   const results = await runOnDataset(chain, datasetName, {\n *     evaluators: [/* ...evaluators... *\\//],\n *     client,\n *   });\n *\n *   console.log('Evaluation Results:', results);\n * }\n *\n * evaluateModel();\n * ```\n * In this example, `runOnDataset` is used to evaluate a language model (or a chain of models) against\n * a dataset named 'example-dataset'. The evaluation process is configured using `RunOnDatasetParams[\"evaluators\"]`, which can\n * include both standard and custom evaluators. The `Client` instance is used to interact with LangChain services.\n * The function returns the evaluation results, which can be logged or further processed as needed.\n */\n\nexport async function runOnDataset(\n  chainOrFactory: ChainOrFactory,\n  datasetName: string,\n  options?: RunOnDatasetParams\n) {\n  const {\n    projectName,\n    projectMetadata,\n    client,\n    maxConcurrency,\n  }: RunOnDatasetParams = options ?? {};\n\n  const evaluationConfig: RunEvalConfig | undefined =\n    options?.evaluationConfig ??\n    (options?.evaluators != null\n      ? {\n          evaluators: options.evaluators,\n          formatEvaluatorInputs: options.formatEvaluatorInputs,\n        }\n      : undefined);\n\n  const wrappedModel = await createWrappedModel(chainOrFactory);\n  const testClient = client ?? new Client();\n  const testProjectName = projectName ?? randomName();\n  const dataset = await testClient.readDataset({ datasetName });\n  const datasetId = dataset.id;\n  const testConcurrency = maxConcurrency ?? 5;\n  const { configs, examples, runExtractors } = await loadExamples({\n    datasetName,\n    client: testClient,\n    projectName: testProjectName,\n    maxConcurrency: testConcurrency,\n  });\n\n  await testClient.createProject({\n    projectName: testProjectName,\n    referenceDatasetId: datasetId,\n    projectExtra: { metadata: { ...projectMetadata } },\n  });\n  const wrappedRunnable: Runnable = new RunnableLambda({\n    func: wrappedModel,\n  }).withConfig({ runName: \"evaluationRun\" });\n  const runInputs = getExamplesInputs(\n    examples,\n    chainOrFactory,\n    dataset.data_type\n  );\n  const progress = new ProgressBar({\n    total: runInputs.length,\n    format: \"Predicting: {bar} {percentage}% | {value}/{total}\",\n  });\n  // TODO: Collect the runs as well.\n  await wrappedRunnable\n    .withListeners({\n      onEnd: () => progress.increment(),\n    })\n    // TODO: Insert evaluation inline for immediate feedback.\n    .batch(runInputs, configs, {\n      maxConcurrency,\n      returnExceptions: true,\n    });\n\n  progress.complete();\n  const runs: Run[] = [];\n  for (let i = 0; i < examples.length; i += 1) {\n    runs.push(await runExtractors[i].extract());\n  }\n  let evalResults: Record<\n    string,\n    { run_id: string; execution_time?: number; feedback: Feedback[] }\n  > = {};\n  if (evaluationConfig) {\n    const loadedEvalConfig =\n      await LoadedEvalConfig.fromRunEvalConfig(evaluationConfig);\n    evalResults = await applyEvaluators({\n      evaluation: loadedEvalConfig,\n      runs,\n      examples,\n      client: testClient,\n      maxConcurrency: testConcurrency,\n    });\n  }\n  const results: EvalResults = {\n    projectName: testProjectName,\n    results: evalResults ?? {},\n  };\n  return results;\n}\n\nfunction isLangsmithTraceableFunction(x: unknown): x is AnyTraceableFunction {\n  return typeof x === \"function\" && \"langsmith:traceable\" in x;\n}\n"],"mappings":";;;;;;;;;;;AAsDA,IAAM,uBAAN,MAA2B;CACzB;CAEA;CAEA,cAAc;EACZ,KAAK,eAAe,IAAI,SAAiB,YAAY;GACnD,KAAK,uBAAuB;EAC9B,CAAC;CACH;CAEA,oBACE,QACA,SACA,UACG;EACH,KAAK,qBAAqB,KAAK;CACjC;CAEA,MAAM,UAA2B;EAC/B,OAAO,KAAK;CACd;AACF;AAEA,IAAM,qBAAN,cAAiC,WAAW;CAC1C;CAEA;;CAGA,OAAO;CAEP,cAAc;EACZ,MAAM;EACN,KAAK,aAAa,IAAI,SAAc,YAAY;GAC9C,KAAK,qBAAqB;EAC5B,CAAC;CACH;CAEA,MAAM,WAAW,KAAU;EACzB,KAAK,mBAAmB,GAAG;CAC7B;CAEA,MAAM,UAAwB;EAC5B,OAAO,KAAK;CACd;AACF;;;;AAKA,IAAM,sBAAN,MAAkD;CAChD;CAEA,YAAY,WAA6B;EACvC,KAAK,YAAY,IAAI,eAAe,EAAE,MAAM,UAAU,CAAC;CACzD;;;;;;;CAQA,MAAM,YAAY,KAAU,SAA8C;EACxE,MAAM,YAAY,IAAI,qBAAqB;EAC3C,MAAM,SAAS,IAAI,gBAAgB,EAAE,aAAa,aAAa,CAAC;EAChE,MAAM,SAAS,MAAM,KAAK,UAAU,OAClC;GACE;GACA;GACA,OAAO,IAAI;GACX,YAAY,IAAI;GAChB,WAAW,SAAS;EACtB,GACA,EACE,WAAW,CAAC,WAAW,MAAM,EAC/B,CACF;EAEA,OAAO;GACL,aAAa,MAFK,UAAU,QAAQ;GAGpC,GAAG;EACL;CACF;AACF;AAGA,SAAS,qBAAqB,WAAiD;CAC7E,OAAO,aAAa,OAAO,UAAU,oBAAoB;AAC3D;;;;;;;;AAYA,IAAM,yBAAN,MAAM,+BAA+B,QAAQ;CAC3C;CAEA,wBAAgE,KAAA;CAEhE,YAAY,QAAuB,iBAAkC;EACnE,MAAM,MAAM;EAEZ,KAAK,kBAAkB;CACzB;CAEA,YAAY,QAA+C;EACzD,MAAM,QAAQ,IAAI,uBAChB;GACE,GAAG;GACH,YAAY;GACZ,cAAc,KAAK;GACnB,QAAQ,KAAK;EACf,GACA,KAAK,uBAAuB,SAAS,KAAK,KAAK,eACjD;EACA,KAAK,WAAW,KAAK,KAAK;EAC1B,OAAO;CACT;CAEA,MAAM,UAAyB;EAE7B,KAAK,wBAAwB,MAAM,KAAK,gBAAgB,iBACtD,OAAO,KAAK,eAAe,YACzB,KAAK,cAAc,QACnB,QAAQ,KAAK,aACX,KAAK,aACL;GACE,IAAI;IAAC;IAAa;IAAS;GAAwB;GACnD,IAAI;GACJ,MAAM;EACR,GACJ,KAAK,QACL,KAAK,IACL,KAAK,UACL,KAAA,GACA,KAAA,GACA,KAAK,IACP;CACF;CAEA,MAAM,WAA0B;EAC9B,IAAI,KAAK,OACP,MAAM,KAAK,uBAAuB,iBAChC,KAAK,OACL,KAAK,IACL,KAAK,YAAY,IACjB,KAAA,GACA,KAAA,CACF;OAEA,MAAM,KAAK,uBAAuB,eAChC,KAAK,WAAW,CAAC,GACjB,KAAK,IACL,KAAK,YAAY,IACjB,KAAA,GACA,KAAA,CACF;CAEJ;AACF;AAEA,IAAM,oBAAN,cAAqD,SAGnD;CACA,kBAAkB;CAElB,eAAe,CAAC,kBAAkB,WAAW;CAE7C;CAEA,YAAY,QAAwC;EAClD,MAAM,MAAM;EAEZ,IAAI,CAAC,6BAA6B,OAAO,IAAI,GAC3C,MAAM,IAAI,MACR,0FACF;EAGF,KAAK,OAAO,OAAO;CACrB;CAEA,MAAM,OAAO,OAAiB,SAAmC;EAC/D,MAAM,CAAC,UAAU,KAAK,gBAAgB,WAAW,CAAC,GAAG,CAAC;EACtD,MAAM,kBAAkB,MAAM,4BAA4B,MAAM;EAEhE,MAAM,gBACJ,yBAAyB,KAAK,OACzB,KAAK,KAAK,yBACX,EAAE,MAAM,WAAW;EAEzB,IAAI,CAAC,iBAAiB,MAAM,IAAI,MAAM,2BAA2B;EACjE,MAAM,UAAU,IAAI,uBAClB;GACE,GAAG;GACH,YAAY,iBAAiB,eACzB,IAAI,QAAQ;IAAE,MAAM;IAAY,IAAI,iBAAiB;GAAa,CAAC,IACnE,KAAA;EACN,GACA,eACF;EAEA,IACE,OAAO,UAAU,YACjB,SAAS,QACT,OAAO,KAAK,KAAK,CAAC,CAAC,WAAW,GAC9B;GACA,IAAI,UAAU,SAAS,MAAM,QAAQ,KAAK,GACxC,OAAQ,MAAM,KAAK,KAAK,SAAS,GAAG,KAAK;GAG3C,IACE,WAAW,SACX,EACE,OAAO,UAAU,YACjB,SAAS,QACT,CAAC,MAAM,QAAQ,KAAK,KAEpB,EAAE,iBAAiB,QAGrB,IAAI;IACF,OAAQ,MAAM,KAAK,KAAK,SAAS,MAAM,KAAK;GAC9C,QAAQ;IACN,OAAQ,MAAM,KAAK,KAAK,SAAS,KAAK;GACxC;EAEJ;EAEA,OAAQ,MAAM,KAAK,KAAK,SAAS,KAAK;CACxC;AACF;;;;;;AAOA,IAAM,uBAAN,MAAM,qBAA6C;CACjD;CAEA;CAEA;CAEA;CAEA,YACE,WACA,gBACA,uBACA;EACA,KAAK,YAAY;EACjB,KAAK,oBAAoB,OAAO,WAAW,oBAAoB;EAC/D,KAAK,iBAAiB;EACtB,KAAK,wBAAwB;CAC/B;CAEA,aAAa,eACX,QAC+B;EAC/B,MAAM,gBACJ,OAAO,WAAW,WAAW,SAAS,OAAO;EAC/C,MAAM,aAAa,OAAO,WAAW,WAAY,CAAC,IAAmB;EACrE,MAAM,YAAY,MAAM,cAAc,eAAe,UAAU;EAC/D,MAAM,cAAc,YAAY,eAAe,WAAW;EAC1D,IAAI,CAAC,qBAAqB,SAAS,GACjC,MAAM,IAAI,MACR,qBAAqB,cAAc,6GAGrC;EAEF,IAAI,CAAC,aACH,MAAM,IAAI,MACR,qBAAqB,cAAc,sGAErC;EAEF,OAAO,IAAI,qBACT,WACA,aACA,YAAY,qBACd;CACF;;;;;;;CAQA,MAAM,YAAY,KAAU,SAA8C;EACxE,MAAM,EAAE,YAAY,OAAO,cAAc,KAAK,sBAAsB;GAClE,UAAU,IAAI;GACd,eAAe,IAAI;GACnB,oBAAoB,SAAS;GAC7B;EACF,CAAC;EACD,MAAM,YAAY,IAAI,qBAAqB;EAC3C,MAAM,SAAS,IAAI,gBAAgB,EAAE,aAAa,aAAa,CAAC;EAChE,IAAI,KAAK,mBAAmB;GAC1B,MAAM,aAAa,MAAM,KAAK,UAAU,gBACtC;IACc;IACD;IACJ;GACT,GACA,EACE,WAAW,CAAC,WAAW,MAAM,EAC/B,CACF;GACA,MAAM,QAAQ,MAAM,UAAU,QAAQ;GACtC,OAAO;IACL,KAAK,KAAK;IACV,SAAS,YAAY;IACrB,aAAa;IACb,GAAG;GACL;EACF;EACA,MAAM,IAAI,MACR,uHAGF;CACF;AACF;AAEA,IAAM,mBAAN,MAAM,iBAAiB;CACF;CAAnB,YAAY,YAA2D;EAApD,KAAA,aAAA;CAAqD;CAExE,aAAa,kBACX,QAC2B;EAE3B,MAAM,oBACJ,QAAQ,oBAAoB,OAAO,YAAY,OAAO,iBAAiB,EAAA,EACtE,KAAK,cAAc;GACpB,IAAI,OAAO,cAAc,YACvB,OAAO,IAAI,oBAAoB,SAAS;QAExC,OAAO;EAEX,CAAC;EAED,MAAM,wBAAwB,MAAM,QAAQ,IAC1C,QAAQ,YACJ,OAAO,sBAAsB,CAAC,EAC9B,IACA,OAAO,cACL,MAAM,qBAAqB,eAAe,SAAS,CACvD,KAAK,CAAC,CACV;EACA,OAAO,IAAI,kBACR,oBAAoB,CAAC,EAAA,CAAG,OAAO,yBAAyB,CAAC,CAAC,CAC7D;CACF;AACF;;;;;;;;;AAwCA,MAAM,qBAAqB,OAAO,mBAAmC;CACnE,IAAI,SAAS,WAAW,cAAc,GACpC,aAAa;CAEf,IAAI,OAAO,mBAAmB,YAAY;EACxC,IAAI,6BAA6B,cAAc,GAAG;GAChD,MAAM,eAAe,IAAI,kBAAkB,EAAE,MAAM,eAAe,CAAC;GACnE,aAAa;EACf;EAEA,IAAI;GAEF,IAAI,MAAO,eAAkC;GAC7C,IACE,OACA,OAAQ,IAAqC,SAAS,YAEtD,MAAM,MAAM;GAEd,OAAO;EACT,QAAQ;GAGN,MAAM,eAAe,IAAI,eAAe,EAAE,MAAM,eAAe,CAAC;GAChE,aAAa;EACf;CACF;CACA,MAAM,IAAI,MAAM,wBAAwB;AAC1C;AAEA,MAAM,eAAe,OAAO,EAC1B,aACA,QACA,kBAMI;CACJ,MAAM,kBAAkB,OAAO,aAAa,EAAE,YAAY,CAAC;CAC3D,MAAM,UAA4B,CAAC;CACnC,MAAM,gBAAgB,CAAC;CACvB,MAAM,WAAW,CAAC;CAClB,WAAW,MAAM,WAAW,iBAAiB;EAC3C,MAAM,eAAe,IAAI,mBAAmB;EAC5C,QAAQ,KAAK,EACX,WAAW,CACT,IAAI,gBAAgB;GAAE,WAAW,QAAQ;GAAI;EAAY,CAAC,GAC1D,YACF,EACF,CAAC;EACD,SAAS,KAAK,OAAO;EACrB,cAAc,KAAK,YAAY;CACjC;CACA,OAAO;EACL;EACA;EACA;CACF;AACF;AAEA,MAAM,kBAAkB,OAAO,EAC7B,YACA,MACA,UACA,QACA,qBAaI;CAEJ,MAAM,EAAE,eAAe;CACvB,MAAM,WAAW,IAAI,YAAY;EAC/B,OAAO,SAAS;EAChB,QAAQ;CACV,CAAC;CACD,MAAM,SAAS,IAAI,YAAY,EAC7B,eACF,CAAC;CACD,MAAM,WAAW,KAAK,IACpB,OACE,KACA,MAMA,OAAO,KAAK,YAAY;EACtB,MAAM,mBAAmB,MAAM,QAAQ,WACrC,WAAW,KAAK,cACd,OAAO,YAAY,KAAK,WAAW;GACjC,kBAAkB,SAAS;GAC3B,eAAe;EACjB,CAAC,CACH,CACF;EACA,SAAS,UAAU;EACnB,OAAO;GACL,gBACE,KAAK,YAAY,IAAI,aACjB,IAAI,KAAK,IAAI,QAAQ,CAAC,CAAC,QAAQ,IAC/B,IAAI,KAAK,IAAI,UAAU,CAAC,CAAC,QAAQ,IACjC,KAAA;GACN,UAAU,iBAAiB,KAAK,eAC9B,WAAW,WAAW,cAClB,WAAW,QACX,WAAW,MACjB;GACA,QAAQ,IAAI;EACd;CACF,CAAC,CACL;CAGA,QAAO,MAFe,QAAQ,IAAI,QAAQ,EAAA,CAE3B,QACZ,KAAK,QAAQ,OAAO;EACnB,GAAG;GACF,SAAS,EAAE,CAAC,KAAK;CACpB,IACA,CAAC,CACH;AACF;AAaA,MAAM,qBACJ,UACA,gBACA,aACG;CACH,IAAI,aAAa,QAIf,OAAO,SAAS,KAAK,EAAE,aACrB,gCAAgC,OAAO,KAAK,CAC9C;CAOF,IAFE,OAAO,mBAAmB,YAC1B,OAAQ,eAAqC,aAAa,cAG1D,SAAS,OAAO,EAAE,aAAa,OAAO,KAAK,MAAM,CAAC,CAAC,WAAW,CAAC,GAE/D,OAAO,SAAS,KAAK,EAAE,aAAa,OAAO,OAAO,MAAM,CAAC,CAAC,EAAE;CAE9D,OAAO,SAAS,KAAK,EAAE,aAAa,MAAM;AAC5C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAqDA,eAAsB,aACpB,gBACA,aACA,SACA;CACA,MAAM,EACJ,aACA,iBACA,QACA,mBACsB,WAAW,CAAC;CAEpC,MAAM,mBACJ,SAAS,qBACR,SAAS,cAAc,OACpB;EACE,YAAY,QAAQ;EACpB,uBAAuB,QAAQ;CACjC,IACA,KAAA;CAEN,MAAM,eAAe,MAAM,mBAAmB,cAAc;CAC5D,MAAM,aAAa,UAAU,IAAI,OAAO;CACxC,MAAM,kBAAkB,eAAe,WAAW;CAClD,MAAM,UAAU,MAAM,WAAW,YAAY,EAAE,YAAY,CAAC;CAC5D,MAAM,YAAY,QAAQ;CAC1B,MAAM,kBAAkB,kBAAkB;CAC1C,MAAM,EAAE,SAAS,UAAU,kBAAkB,MAAM,aAAa;EAC9D;EACA,QAAQ;EACR,aAAa;EACb,gBAAgB;CAClB,CAAC;CAED,MAAM,WAAW,cAAc;EAC7B,aAAa;EACb,oBAAoB;EACpB,cAAc,EAAE,UAAU,EAAE,GAAG,gBAAgB,EAAE;CACnD,CAAC;CACD,MAAM,kBAA4B,IAAI,eAAe,EACnD,MAAM,aACR,CAAC,CAAC,CAAC,WAAW,EAAE,SAAS,gBAAgB,CAAC;CAC1C,MAAM,YAAY,kBAChB,UACA,gBACA,QAAQ,SACV;CACA,MAAM,WAAW,IAAI,YAAY;EAC/B,OAAO,UAAU;EACjB,QAAQ;CACV,CAAC;CAED,MAAM,gBACH,cAAc,EACb,aAAa,SAAS,UAAU,EAClC,CAAC,CAAC,CAED,MAAM,WAAW,SAAS;EACzB;EACA,kBAAkB;CACpB,CAAC;CAEH,SAAS,SAAS;CAClB,MAAM,OAAc,CAAC;CACrB,KAAK,IAAI,IAAI,GAAG,IAAI,SAAS,QAAQ,KAAK,GACxC,KAAK,KAAK,MAAM,cAAc,EAAE,CAAC,QAAQ,CAAC;CAE5C,IAAI,cAGA,CAAC;CACL,IAAI,kBAAkB;EACpB,MAAM,mBACJ,MAAM,iBAAiB,kBAAkB,gBAAgB;EAC3D,cAAc,MAAM,gBAAgB;GAClC,YAAY;GACZ;GACA;GACA,QAAQ;GACR,gBAAgB;EAClB,CAAC;CACH;CAKA,OAAO;EAHL,aAAa;EACb,SAAS,eAAe,CAAC;CAEd;AACf;AAEA,SAAS,6BAA6B,GAAuC;CAC3E,OAAO,OAAO,MAAM,cAAc,yBAAyB;AAC7D"}