@ekairos/dataset 1.8.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/clearDataset.tool.d.ts +22 -0
- package/dist/clearDataset.tool.d.ts.map +1 -0
- package/dist/clearDataset.tool.js +58 -0
- package/dist/clearDataset.tool.js.map +1 -0
- package/dist/completeDataset.tool.d.ts +27 -0
- package/dist/completeDataset.tool.d.ts.map +1 -0
- package/dist/completeDataset.tool.js +258 -0
- package/dist/completeDataset.tool.js.map +1 -0
- package/dist/datasetFiles.d.ts +4 -0
- package/dist/datasetFiles.d.ts.map +1 -0
- package/dist/datasetFiles.js +13 -0
- package/dist/datasetFiles.js.map +1 -0
- package/dist/domain.d.ts +2 -0
- package/dist/domain.d.ts.map +1 -0
- package/dist/domain.js +6 -0
- package/dist/domain.js.map +1 -0
- package/dist/executeCommand.tool.d.ts +37 -0
- package/dist/executeCommand.tool.d.ts.map +1 -0
- package/dist/executeCommand.tool.js +123 -0
- package/dist/executeCommand.tool.js.map +1 -0
- package/dist/file/file-dataset.agent.d.ts +51 -0
- package/dist/file/file-dataset.agent.d.ts.map +1 -0
- package/dist/file/file-dataset.agent.js +272 -0
- package/dist/file/file-dataset.agent.js.map +1 -0
- package/dist/file/filepreview.d.ts +41 -0
- package/dist/file/filepreview.d.ts.map +1 -0
- package/dist/file/filepreview.js +188 -0
- package/dist/file/filepreview.js.map +1 -0
- package/dist/file/generateSchema.tool.d.ts +31 -0
- package/dist/file/generateSchema.tool.d.ts.map +1 -0
- package/dist/file/generateSchema.tool.js +108 -0
- package/dist/file/generateSchema.tool.js.map +1 -0
- package/dist/file/index.d.ts +2 -0
- package/dist/file/index.d.ts.map +1 -0
- package/dist/file/index.js +18 -0
- package/dist/file/index.js.map +1 -0
- package/dist/file/prompts.d.ts +3 -0
- package/dist/file/prompts.d.ts.map +1 -0
- package/dist/file/prompts.js +194 -0
- package/dist/file/prompts.js.map +1 -0
- package/dist/file/scripts/file_metadata.py +71 -0
- package/dist/file/scripts/preview_head_csv.py +24 -0
- package/dist/file/scripts/preview_head_excel.py +30 -0
- package/dist/file/scripts/preview_mid_csv.py +36 -0
- package/dist/file/scripts/preview_mid_excel.py +39 -0
- package/dist/file/scripts/preview_tail_csv.py +27 -0
- package/dist/file/scripts/preview_tail_excel.py +29 -0
- package/dist/index.d.ts +8 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +24 -0
- package/dist/index.js.map +1 -0
- package/dist/schema.d.ts +62 -0
- package/dist/schema.d.ts.map +1 -0
- package/dist/schema.js +46 -0
- package/dist/schema.js.map +1 -0
- package/dist/service.d.ts +73 -0
- package/dist/service.d.ts.map +1 -0
- package/dist/service.js +423 -0
- package/dist/service.js.map +1 -0
- package/dist/transform/filepreview.d.ts +24 -0
- package/dist/transform/filepreview.d.ts.map +1 -0
- package/dist/transform/filepreview.js +92 -0
- package/dist/transform/filepreview.js.map +1 -0
- package/dist/transform/index.d.ts +2 -0
- package/dist/transform/index.d.ts.map +1 -0
- package/dist/transform/index.js +18 -0
- package/dist/transform/index.js.map +1 -0
- package/dist/transform/prompts.d.ts +35 -0
- package/dist/transform/prompts.d.ts.map +1 -0
- package/dist/transform/prompts.js +137 -0
- package/dist/transform/prompts.js.map +1 -0
- package/dist/transform/transform-dataset.agent.d.ts +58 -0
- package/dist/transform/transform-dataset.agent.d.ts.map +1 -0
- package/dist/transform/transform-dataset.agent.js +283 -0
- package/dist/transform/transform-dataset.agent.js.map +1 -0
- package/package.json +56 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"service.js","sourceRoot":"","sources":["../src/service.ts"],"names":[],"mappings":";;;AAAA,4CAA0E;AAE1E,qCAAyC;AAQzC,MAAa,cAAc;IAGvB;QACI,IAAI,CAAC,EAAE,GAAG,IAAA,YAAI,EAAC;YACX,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,0BAAoC;YACvD,UAAU,EAAE,OAAO,CAAC,GAAG,CAAC,uBAAiC;YACzD,MAAM,EAAE,sBAAa,CAAC,MAAM,EAAuB;SACtD,CAAC,CAAA;IACN,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,MAOnB;QACG,IAAI,CAAC;YACD,MAAM,SAAS,GAAG,MAAM,CAAC,EAAE,IAAI,IAAA,UAAK,GAAE,CAAA;YACtC,MAAM,SAAS,GAAG,EAAE,CAAA;YAEpB,SAAS,CAAC,IAAI,CACV,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC;gBAC1C,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,EAAE;gBAC7B,YAAY,EAAE,MAAM,CAAC,YAAY,IAAI,EAAE;gBACvC,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,SAAS;gBAClC,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;gBACrB,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;gBACrB,GAAG,MAAM;aACZ,CAAC,CACL,CAAA;YAED,IAAI,MAAM,CAAC,cAAc,EAAE,CAAC;gBACxB,SAAS,CAAC,IAAI,CACV,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC;oBACxC,YAAY,EAAE,MAAM,CAAC,cAAc;iBACtC,CAAC,CACL,CAAA;YACL,CAAC;YAED,MAAM,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAA;YAEjC,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,SAAS,EAAE,EAAE,CAAA;QAC5C,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACX,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YACtE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,CAAA;QACxC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,SAAiB,EAAE,OAA4B;QAC/D,IAAI,CAAC;YACD,MAAM,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC;gBACnB,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC;oBAC1C,GAAG,OAAO;oBACV,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;iBACxB,CAAC;aACL,CAAC,CAAA;YAEF,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,CAAA;QACxC,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACX,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YACtE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,CAAA;QACxC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAC,MAGvB;QACG,IAAI,CAAC;YACD,MAAM,SAAS,GAAU,EAAE,CAAA;YAE3B,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;gBAClC,MAAM,QAAQ,GAAG,IAAA,UAAK,GAAE,CAAA;gBACxB,SAAS,CAAC,IAAI,CACV,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC;oBACxC,UAAU,EAAE,MAAM,CAAC,UAAU;oBAC7B,KAAK,EAAE,MAAM,CAAC,KAAK;oBACnB,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;iBACxB,CAAC,EACF,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,gBAAgB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAC9E,CAAA;YACL,CAAC;YAED,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACvB,MAAM,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAA;YACrC,CAAC;YAED,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,UAAU,EAAE,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAA;QACpE,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACX,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YACtE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,CAAA;QACxC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,sBAAsB,CAAC,MAI5B;QACG,IAAI,CAAC;YACD,MAAM,cAAc,GAAG,CAAC,GAAG,MAAM,CAAC,cAAc,CAAC,CAAA;YAEjD,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;gBAC1B,cAAc,CAAC,IAAI,CACf,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,gBAAgB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC;oBACjD,MAAM,EAAE,WAAW;oBACnB,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;oBACrB,uBAAuB,EAAE,MAAM,CAAC,gBAAgB,CAAC,cAAc,IAAI,CAAC;iBACvE,CAAC,CACL,CAAA;YACL,CAAC;YAED,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5B,MAAM,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAA;YAC1C,CAAC;YAED,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,CAAA;QACxC,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACX,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YACtE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,CAAA;QACxC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,MAAc;QACpC,IAAI,CAAC;YACD,MAAM,YAAY,GAAQ,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC;gBAC1C,gBAAgB,EAAE;oBACd,CAAC,EAAE;wBACC,KAAK,EAAE;4BACH,UAAU,EAAE,MAAM;yBACd;wBACR,KAAK,EAAE,CAAC;qBACX;iBACJ;aACJ,CAAC,CAAA;YAEF,OAAO,YAAY,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,CAAA;QACrD,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACX,OAAO,CAAC,KAAK,CAAC,mCAAmC,EAAE,KAAK,CAAC,CAAA;YACzD,OAAO,IAAI,CAAA;QACf,CAAC;IACL,CAAC;IAED,KAAK,CAAC,kBAAkB,CAAC,SAAiB;QACtC,IAAI,CAAC;YACD,MAAM,kBAAkB,GAAQ,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC;gBAChD,eAAe,EAAE;oBACb,CAAC,EAAE;wBACC,KAAK,EAAE;4BACH,YAAY,EAAE,SAAS;yBACnB;wBACR,KAAK,EAAE,CAAC;wBACR,MAAM,EAAE,CAAC,IAAI,CAAC;qBACjB;iBACJ;aACJ,CAAC,CAAA;YAEF,OAAO,kBAAkB,CAAC,eAAe,IAAI,EAAE,CAAA;QACnD,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACX,OAAO,CAAC,KAAK,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAA;YACtD,OAAO,EAAE,CAAA;QACb,CAAC;IACL,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,MAAc;QAC5B,IAAI,CAAC;YACD,MAAM,SAAS,GAAQ,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC;gBACvC,MAAM,EAAE;oBACJ,CAAC,EAAE;wBACC,KAAK,EAAE;4BACH,EAAE,EAAE,MAAM;yBACN;wBACR,KAAK,EAAE,CAAC;qBACX;iBACJ;aACJ,CAAC,CAAA;YAEF,OAAO,SAAS,CAAA;QACpB,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACX,OAAO,CAAC,KAAK,CAAC,2BAA2B,EAAE,KAAK,CAAC,CAAA;YACjD,MAAM,KAAK,CAAA;QACf,CAAC;IACL,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,SAAiB;QAClC,IAAI,CAAC;YACD,MAAM,KAAK,GAAQ,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC;gBACnC,gBAAgB,EAAE;oBACd,CAAC,EAAE;wBACC,KAAK,EAAE,EAAE,EAAE,EAAE,SAAS,EAAE;wBACxB,KAAK,EAAE,CAAC;qBACX;iBACJ;aACJ,CAAC,CAAA;YAEF,MAAM,OAAO,GAAG,KAAK,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,CAAA;YAC3C,IAAI,CAAC,OAAO,EAAE,CAAC;gBACX,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,8BAA8B,SAAS,EAAE,EAAE,CAAA;YAC1E,CAAC;YAED,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,CAAA;QACtC,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACX,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YACtE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,CAAA;QACxC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,MAIzB;QACG,IAAI,CAAC;YACD,MAAM,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC;gBACnB,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,gBAAgB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC;oBACjD,MAAM,EAAE,MAAM,CAAC,MAAM;oBACrB,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,iBAAiB;oBAC1C,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;iBACxB,CAAC;aACL,CAAC,CAAA;YAEF,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,CAAA;QACxC,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACX,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YACtE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,CAAA;QACxC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,MAKzB;QACG,IAAI,CAAC;YACD,MAAM,OAAO,GAAQ;gBACjB,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;aACxB,CAAA;YAED,IAAI,MAAM,CAAC,mBAAmB,KAAK,SAAS,EAAE,CAAC;gBAC3C,OAAO,CAAC,mBAAmB,GAAG,MAAM,CAAC,mBAAmB,CAAA;YAC5D,CAAC;YAED,IAAI,MAAM,CAAC,uBAAuB,KAAK,SAAS,EAAE,CAAC;gBAC/C,OAAO,CAAC,uBAAuB,GAAG,MAAM,CAAC,uBAAuB,CAAA;YACpE,CAAC;YAED,MAAM,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC;gBACnB,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,gBAAgB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC;aAChE,CAAC,CAAA;YAEF,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,CAAA;QACxC,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACX,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YACtE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,CAAA;QACxC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,4BAA4B,CAAC,SAAiB;QAChD,IAAI,CAAC;YACD,MAAM,KAAK,GAAQ,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC;gBACnC,eAAe,EAAE;oBACb,CAAC,EAAE;wBACC,KAAK,EAAE;4BACH,YAAY,EAAE,SAAS;yBACnB;qBACX;iBACJ;aACJ,CAAC,CAAA;YAEF,MAAM,OAAO,GAAG,KAAK,CAAC,eAAe,IAAI,EAAE,CAAA;YAC3C,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,CAAA;QACtC,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACX,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YACtE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,CAAA;QACxC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,yBAAyB,CAAC,SAAmB;QAC/C,IAAI,CAAC;YACD,MAAM,EAAE,GAAU,EAAE,CAAA;YACpB,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;gBAC/B,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC,MAAM,EAAE,CAAC,CAAA;YAC1D,CAAC;YAED,MAAM,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAA;YAC1B,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,CAAC,MAAM,EAAE,CAAA;QAC/C,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACX,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YACtE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,CAAA;QACxC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,SAAiB;QAChC,IAAI,CAAC;YACD,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,4BAA4B,CAAC,SAAS,CAAC,CAAA;YACxE,IAAI,CAAC,aAAa,CAAC,EAAE,EAAE,CAAC;gBACpB,OAAO,aAAa,CAAA;YACxB,CAAC;YAED,MAAM,OAAO,GAAG,aAAa,CAAC,IAAI,CAAA;YAClC,MAAM,UAAU,GAAG,GAAG,CAAA;YACtB,IAAI,YAAY,GAAG,CAAC,CAAA;YAEpB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,IAAI,UAAU,EAAE,CAAC;gBAClD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,CAAA;gBAC9C,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;gBACvC,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,yBAAyB,CAAC,QAAQ,CAAC,CAAA;gBAEnE,IAAI,CAAC,YAAY,CAAC,EAAE,EAAE,CAAC;oBACnB,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,mCAAmC,CAAC,KAAK,YAAY,CAAC,KAAK,EAAE,EAAE,CAAA;gBAC9F,CAAC;gBAED,YAAY,IAAI,YAAY,CAAC,IAAI,CAAA;YACrC,CAAC;YAED,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC;gBAChD,SAAS;gBACT,MAAM,EAAE,SAAS;gBACjB,uBAAuB,EAAE,CAAC;gBAC1B,mBAAmB,EAAE,CAAC;aACzB,CAAC,CAAA;YAEF,IAAI,CAAC,YAAY,CAAC,EAAE,EAAE,CAAC;gBACnB,OAAO,YAAY,CAAA;YACvB,CAAC;YAED,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,YAAY,EAAE,EAAE,CAAA;QAC/C,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACX,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YACtE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,CAAA;QACxC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,uBAAuB,CAAC,MAG7B;QACG,IAAI,CAAC;YACD,MAAM,WAAW,GAAG,YAAY,MAAM,CAAC,SAAS,eAAe,CAAA;YAE/D,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC,WAAW,EAAE,MAAM,CAAC,UAAU,EAAE;gBAClF,WAAW,EAAE,sBAAsB;gBACnC,kBAAkB,EAAE,cAAc;aACrC,CAAC,CAAA;YAEF,IAAI,CAAC,YAAY,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;gBAC1B,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,kCAAkC,EAAE,CAAA;YACnE,CAAC;YAED,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC;gBAC5C,SAAS,EAAE,MAAM,CAAC,SAAS;gBAC3B,MAAM,EAAE,YAAY,CAAC,IAAI,CAAC,EAAE;gBAC5B,WAAW;aACd,CAAC,CAAA;YAEF,IAAI,CAAC,UAAU,CAAC,EAAE,EAAE,CAAC;gBACjB,OAAO,UAAU,CAAA;YACrB,CAAC;YAED,OAAO;gBACH,EAAE,EAAE,IAAI;gBACR,IAAI,EAAE;oBACF,MAAM,EAAE,YAAY,CAAC,IAAI,CAAC,EAAE;oBAC5B,WAAW;iBACd;aACJ,CAAA;QACL,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACX,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YACtE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,CAAA;QACxC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAC,MAIvB;QACG,IAAI,CAAC;YACD,MAAM,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC;gBACnB,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,gBAAgB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC;aAClF,CAAC,CAAA;YAEF,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,CAAA;QACxC,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACX,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YACtE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,CAAA;QACxC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,SAAiB;QACvC,IAAI,CAAC;YACD,MAAM,SAAS,GAAQ,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC;gBACvC,gBAAgB,EAAE;oBACd,CAAC,EAAE;wBACC,KAAK,EAAE,EAAE,EAAE,EAAE,SAAS,EAAE;wBACxB,KAAK,EAAE,CAAC;qBACX;oBACD,QAAQ,EAAE,EAAE;iBACR,EAAE,iBAAiB;aAC9B,CAAC,CAAA;YAEF,MAAM,aAAa,GAAG,SAAS,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,CAAA;YACrD,MAAM,QAAQ,GAAG,aAAa,EAAE,QAAQ,CAAA;YACxC,MAAM,UAAU,GAAG,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAA;YAEnE,IAAI,CAAC,UAAU,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC;gBACjC,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,+BAA+B,EAAE,CAAA;YAChE,CAAC;YAED,KAAK,SAAS,CAAC,CAAC,eAAe,CAAC,GAAW;gBACvC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAA;gBAEjC,IAAI,CAAC,QAAQ,CAAC,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;oBACjC,MAAM,IAAI,KAAK,CAAC,2CAA2C,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAA;gBACjF,CAAC;gBAED,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,CAAA;gBACxC,MAAM,OAAO,GAAG,IAAI,WAAW,CAAC,OAAO,CAAC,CAAA;gBACxC,IAAI,MAAM,GAAG,EAAE,CAAA;gBAEf,OAAO,IAAI,EACX,CAAC;oBACG,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAA;oBAC3C,IAAI,IAAI,EAAE,CAAC;wBACP,MAAK;oBACT,CAAC;oBAED,MAAM,IAAI,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;oBACjD,IAAI,YAAY,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;oBAEvC,OAAO,YAAY,KAAK,CAAC,CAAC,EAC1B,CAAC;wBACG,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,YAAY,CAAC,CAAA;wBAC1C,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,YAAY,GAAG,CAAC,CAAC,CAAA;wBAEvC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAA;wBAC3B,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;4BACvB,YAAY,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;4BACnC,SAAQ;wBACZ,CAAC;wBAED,IAAI,MAAW,CAAA;wBACf,IAAI,CAAC;4BACD,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAA;wBAChC,CAAC;wBACD,OAAO,KAAK,EACZ,CAAC;4BACG,OAAO,CAAC,KAAK,CAAC,qCAAqC,EAAE,KAAK,CAAC,CAAA;4BAC3D,YAAY,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;4BACnC,SAAQ;wBACZ,CAAC;wBAED,IAAI,MAAM,IAAI,MAAM,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;4BAClC,MAAM,EAAE,UAAU,EAAE,MAAM,CAAC,IAAI,EAAE,CAAA;wBACrC,CAAC;wBAED,YAAY,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;oBACvC,CAAC;gBACL,CAAC;gBAED,MAAM,IAAI,OAAO,CAAC,MAAM,EAAE,CAAA;gBAC1B,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,EAAE,CAAA;gBAC7B,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACrB,IAAI,CAAC;wBACD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAA;wBAClC,IAAI,MAAM,IAAI,MAAM,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;4BAClC,MAAM,EAAE,UAAU,EAAE,MAAM,CAAC,IAAI,EAAE,CAAA;wBACrC,CAAC;oBACL,CAAC;oBACD,OAAO,KAAK,EACZ,CAAC;wBACG,OAAO,CAAC,KAAK,CAAC,qCAAqC,EAAE,KAAK,CAAC,CAAA;oBAC/D,CAAC;gBACL,CAAC;YACL,CAAC;YAED,MAAM,SAAS,GAAG,eAAe,CAAC,UAAU,CAAC,GAAa,CAAC,CAAA;YAC3D,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,CAAA;QACxC,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACX,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YACtE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,CAAA;QACxC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,qBAAqB,CAAC,SAAiB;QACzC,IAAI,CAAC;YACD,MAAM,KAAK,GAAQ,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC;gBACnC,cAAc,EAAE;oBACZ,CAAC,EAAE;wBACC,KAAK,EAAE,EAAE,mBAAmB,EAAE,SAAS,EAAE;wBACzC,KAAK,EAAE,CAAC;wBACR,KAAK,EAAE,EAAE,eAAe,EAAE,MAAM,EAAE;qBAC9B;iBACX;aACJ,CAAC,CAAA;YAEF,MAAM,OAAO,GAAG,KAAK,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,CAAA;YACzC,IAAI,CAAC,OAAO,EAAE,CAAC;gBACX,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,kCAAkC,SAAS,EAAE,EAAE,CAAA;YAC9E,CAAC;YAED,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,CAAA;QACtC,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACX,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YACtE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,CAAA;QACxC,CAAC;IACL,CAAC;CACJ;AAjhBD,wCAihBC"}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { Sandbox } from "@vercel/sandbox";
|
|
2
|
+
export type TransformSourcePreviewContext = {
|
|
3
|
+
totalRows: number;
|
|
4
|
+
metadata?: {
|
|
5
|
+
description: string;
|
|
6
|
+
script: string;
|
|
7
|
+
command: string;
|
|
8
|
+
stdout: string;
|
|
9
|
+
stderr: string;
|
|
10
|
+
};
|
|
11
|
+
head?: {
|
|
12
|
+
description: string;
|
|
13
|
+
script: string;
|
|
14
|
+
command: string;
|
|
15
|
+
stdout: string;
|
|
16
|
+
stderr: string;
|
|
17
|
+
};
|
|
18
|
+
};
|
|
19
|
+
interface PreviewOptions {
|
|
20
|
+
headLines?: number;
|
|
21
|
+
}
|
|
22
|
+
export declare function generateSourcePreview(sandbox: Sandbox, sourcePath: string, datasetId: string, options?: PreviewOptions): Promise<TransformSourcePreviewContext>;
|
|
23
|
+
export {};
|
|
24
|
+
//# sourceMappingURL=filepreview.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"filepreview.d.ts","sourceRoot":"","sources":["../../src/transform/filepreview.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAA;AAGzC,MAAM,MAAM,6BAA6B,GAAG;IACxC,SAAS,EAAE,MAAM,CAAA;IACjB,QAAQ,CAAC,EAAE;QACP,WAAW,EAAE,MAAM,CAAA;QACnB,MAAM,EAAE,MAAM,CAAA;QACd,OAAO,EAAE,MAAM,CAAA;QACf,MAAM,EAAE,MAAM,CAAA;QACd,MAAM,EAAE,MAAM,CAAA;KACjB,CAAA;IACD,IAAI,CAAC,EAAE;QACH,WAAW,EAAE,MAAM,CAAA;QACnB,MAAM,EAAE,MAAM,CAAA;QACd,OAAO,EAAE,MAAM,CAAA;QACf,MAAM,EAAE,MAAM,CAAA;QACd,MAAM,EAAE,MAAM,CAAA;KACjB,CAAA;CACJ,CAAA;AAED,UAAU,cAAc;IACpB,SAAS,CAAC,EAAE,MAAM,CAAA;CACrB;AA6CD,wBAAsB,qBAAqB,CACvC,OAAO,EAAE,OAAO,EAChB,UAAU,EAAE,MAAM,EAClB,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE,cAAmB,GAC7B,OAAO,CAAC,6BAA6B,CAAC,CAkFxC"}
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.generateSourcePreview = generateSourcePreview;
|
|
4
|
+
const datasetFiles_1 = require("../datasetFiles");
|
|
5
|
+
const DEFAULT_HEAD_LINES = 50;
|
|
6
|
+
async function runPythonSnippet(sandbox, datasetId, scriptName, code, args, description) {
|
|
7
|
+
const workstation = (0, datasetFiles_1.getDatasetWorkstation)(datasetId);
|
|
8
|
+
const scriptPath = `${workstation}/${scriptName}.py`;
|
|
9
|
+
await sandbox.writeFiles([
|
|
10
|
+
{
|
|
11
|
+
path: scriptPath,
|
|
12
|
+
content: Buffer.from(code, "utf-8"),
|
|
13
|
+
},
|
|
14
|
+
]);
|
|
15
|
+
const result = await sandbox.runCommand({
|
|
16
|
+
cmd: "python",
|
|
17
|
+
args: [scriptPath, ...args],
|
|
18
|
+
});
|
|
19
|
+
const stdout = (await result.stdout()) || "";
|
|
20
|
+
const stderr = (await result.stderr()) || "";
|
|
21
|
+
return {
|
|
22
|
+
description,
|
|
23
|
+
script: code,
|
|
24
|
+
command: `python ${scriptPath} ${args.join(" ")}`,
|
|
25
|
+
stdout,
|
|
26
|
+
stderr,
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
async function generateSourcePreview(sandbox, sourcePath, datasetId, options = {}) {
|
|
30
|
+
const context = {
|
|
31
|
+
totalRows: 0,
|
|
32
|
+
};
|
|
33
|
+
const countScript = `
|
|
34
|
+
import json, sys
|
|
35
|
+
path = sys.argv[1]
|
|
36
|
+
count = 0
|
|
37
|
+
try:
|
|
38
|
+
with open(path, 'r', encoding='utf-8') as f:
|
|
39
|
+
for line in f:
|
|
40
|
+
s = line.strip()
|
|
41
|
+
if not s:
|
|
42
|
+
continue
|
|
43
|
+
try:
|
|
44
|
+
obj = json.loads(s)
|
|
45
|
+
if isinstance(obj, dict) and obj.get('type') == 'row' and 'data' in obj:
|
|
46
|
+
count += 1
|
|
47
|
+
except Exception:
|
|
48
|
+
pass
|
|
49
|
+
print(json.dumps({ 'row_count': count }))
|
|
50
|
+
except Exception as e:
|
|
51
|
+
print(str(e))
|
|
52
|
+
`;
|
|
53
|
+
const meta = await runPythonSnippet(sandbox, datasetId, "jsonl_count", countScript, [sourcePath], "Counts number of JSONL records with type='row'");
|
|
54
|
+
context.metadata = meta;
|
|
55
|
+
try {
|
|
56
|
+
if (meta.stdout) {
|
|
57
|
+
const parsed = JSON.parse(meta.stdout);
|
|
58
|
+
context.totalRows = Number(parsed.row_count || 0);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
catch {
|
|
62
|
+
context.totalRows = 0;
|
|
63
|
+
}
|
|
64
|
+
const headLines = options.headLines || DEFAULT_HEAD_LINES;
|
|
65
|
+
const headScript = `
|
|
66
|
+
import json, sys
|
|
67
|
+
path = sys.argv[1]
|
|
68
|
+
limit = int(sys.argv[2])
|
|
69
|
+
printed = 0
|
|
70
|
+
try:
|
|
71
|
+
with open(path, 'r', encoding='utf-8') as f:
|
|
72
|
+
for line in f:
|
|
73
|
+
if printed >= limit:
|
|
74
|
+
break
|
|
75
|
+
s = line.strip()
|
|
76
|
+
if not s:
|
|
77
|
+
continue
|
|
78
|
+
try:
|
|
79
|
+
obj = json.loads(s)
|
|
80
|
+
if isinstance(obj, dict) and obj.get('type') == 'row' and 'data' in obj:
|
|
81
|
+
print(json.dumps(obj))
|
|
82
|
+
printed += 1
|
|
83
|
+
except Exception:
|
|
84
|
+
pass
|
|
85
|
+
except Exception as e:
|
|
86
|
+
print(str(e))
|
|
87
|
+
`;
|
|
88
|
+
const head = await runPythonSnippet(sandbox, datasetId, "jsonl_head", headScript, [sourcePath, String(headLines)], `Reads the first ${headLines} JSONL row records`);
|
|
89
|
+
context.head = head;
|
|
90
|
+
return context;
|
|
91
|
+
}
|
|
92
|
+
//# sourceMappingURL=filepreview.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"filepreview.js","sourceRoot":"","sources":["../../src/transform/filepreview.ts"],"names":[],"mappings":";;AAoEA,sDAuFC;AA1JD,kDAAuD;AAwBvD,MAAM,kBAAkB,GAAG,EAAE,CAAA;AAE7B,KAAK,UAAU,gBAAgB,CAC3B,OAAgB,EAChB,SAAiB,EACjB,UAAkB,EAClB,IAAY,EACZ,IAAc,EACd,WAAmB;IAQnB,MAAM,WAAW,GAAG,IAAA,oCAAqB,EAAC,SAAS,CAAC,CAAA;IACpD,MAAM,UAAU,GAAG,GAAG,WAAW,IAAI,UAAU,KAAK,CAAA;IAEpD,MAAM,OAAO,CAAC,UAAU,CAAC;QACrB;YACI,IAAI,EAAE,UAAU;YAChB,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC;SACtC;KACJ,CAAC,CAAA;IAEF,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC;QACpC,GAAG,EAAE,QAAQ;QACb,IAAI,EAAE,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC;KAC9B,CAAC,CAAA;IAEF,MAAM,MAAM,GAAG,CAAC,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAA;IAC5C,MAAM,MAAM,GAAG,CAAC,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAA;IAE5C,OAAO;QACH,WAAW;QACX,MAAM,EAAE,IAAI;QACZ,OAAO,EAAE,UAAU,UAAU,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE;QACjD,MAAM;QACN,MAAM;KACT,CAAA;AACL,CAAC;AAEM,KAAK,UAAU,qBAAqB,CACvC,OAAgB,EAChB,UAAkB,EAClB,SAAiB,EACjB,UAA0B,EAAE;IAE5B,MAAM,OAAO,GAAkC;QAC3C,SAAS,EAAE,CAAC;KACf,CAAA;IAED,MAAM,WAAW,GAAG;;;;;;;;;;;;;;;;;;;CAmBvB,CAAA;IAEG,MAAM,IAAI,GAAG,MAAM,gBAAgB,CAC/B,OAAO,EACP,SAAS,EACT,aAAa,EACb,WAAW,EACX,CAAC,UAAU,CAAC,EACZ,gDAAgD,CACnD,CAAA;IACD,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAA;IACvB,IAAI,CAAC;QACD,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACd,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;YACtC,OAAO,CAAC,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,SAAS,IAAI,CAAC,CAAC,CAAA;QACrD,CAAC;IACL,CAAC;IACD,MAAM,CAAC;QACH,OAAO,CAAC,SAAS,GAAG,CAAC,CAAA;IACzB,CAAC;IAED,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,kBAAkB,CAAA;IAEzD,MAAM,UAAU,GAAG;;;;;;;;;;;;;;;;;;;;;;CAsBtB,CAAA;IAEG,MAAM,IAAI,GAAG,MAAM,gBAAgB,CAC/B,OAAO,EACP,SAAS,EACT,YAAY,EACZ,UAAU,EACV,CAAC,UAAU,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC,EAC/B,mBAAmB,SAAS,oBAAoB,CACnD,CAAA;IACD,OAAO,CAAC,IAAI,GAAG,IAAI,CAAA;IAEnB,OAAO,OAAO,CAAA;AAClB,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/transform/index.ts"],"names":[],"mappings":"AAAA,cAAc,2BAA2B,CAAA"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
14
|
+
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
|
+
};
|
|
16
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
+
__exportStar(require("./transform-dataset.agent"), exports);
|
|
18
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/transform/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,4DAAyC"}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
export type TransformPromptContext = {
|
|
2
|
+
datasetId: string;
|
|
3
|
+
sourceDatasetIds: string[];
|
|
4
|
+
outputSchema: any;
|
|
5
|
+
sandboxConfig: {
|
|
6
|
+
sourcePaths: Array<{
|
|
7
|
+
datasetId: string;
|
|
8
|
+
path: string;
|
|
9
|
+
}>;
|
|
10
|
+
outputPath: string;
|
|
11
|
+
};
|
|
12
|
+
sourcePreviews?: Array<{
|
|
13
|
+
datasetId: string;
|
|
14
|
+
preview: {
|
|
15
|
+
totalRows: number;
|
|
16
|
+
metadata?: {
|
|
17
|
+
description: string;
|
|
18
|
+
script: string;
|
|
19
|
+
command: string;
|
|
20
|
+
stdout: string;
|
|
21
|
+
stderr: string;
|
|
22
|
+
};
|
|
23
|
+
head?: {
|
|
24
|
+
description: string;
|
|
25
|
+
script: string;
|
|
26
|
+
command: string;
|
|
27
|
+
stdout: string;
|
|
28
|
+
stderr: string;
|
|
29
|
+
};
|
|
30
|
+
};
|
|
31
|
+
}>;
|
|
32
|
+
errors: string[];
|
|
33
|
+
};
|
|
34
|
+
export declare function buildTransformDatasetPrompt(context: TransformPromptContext): string;
|
|
35
|
+
//# sourceMappingURL=prompts.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"prompts.d.ts","sourceRoot":"","sources":["../../src/transform/prompts.ts"],"names":[],"mappings":"AAEA,MAAM,MAAM,sBAAsB,GAAG;IACjC,SAAS,EAAE,MAAM,CAAA;IACjB,gBAAgB,EAAE,MAAM,EAAE,CAAA;IAC1B,YAAY,EAAE,GAAG,CAAA;IACjB,aAAa,EAAE;QACX,WAAW,EAAE,KAAK,CAAC;YAAE,SAAS,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;QACvD,UAAU,EAAE,MAAM,CAAA;KACrB,CAAA;IACD,cAAc,CAAC,EAAE,KAAK,CAAC;QACnB,SAAS,EAAE,MAAM,CAAA;QACjB,OAAO,EAAE;YACL,SAAS,EAAE,MAAM,CAAA;YACjB,QAAQ,CAAC,EAAE;gBACP,WAAW,EAAE,MAAM,CAAA;gBACnB,MAAM,EAAE,MAAM,CAAA;gBACd,OAAO,EAAE,MAAM,CAAA;gBACf,MAAM,EAAE,MAAM,CAAA;gBACd,MAAM,EAAE,MAAM,CAAA;aACjB,CAAA;YACD,IAAI,CAAC,EAAE;gBACH,WAAW,EAAE,MAAM,CAAA;gBACnB,MAAM,EAAE,MAAM,CAAA;gBACd,OAAO,EAAE,MAAM,CAAA;gBACf,MAAM,EAAE,MAAM,CAAA;gBACd,MAAM,EAAE,MAAM,CAAA;aACjB,CAAA;SACJ,CAAA;KACJ,CAAC,CAAA;IACF,MAAM,EAAE,MAAM,EAAE,CAAA;CACnB,CAAA;AA0ID,wBAAgB,2BAA2B,CAAC,OAAO,EAAE,sBAAsB,GAAG,MAAM,CAYnF"}
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.buildTransformDatasetPrompt = buildTransformDatasetPrompt;
|
|
4
|
+
const xmlbuilder2_1 = require("xmlbuilder2");
|
|
5
|
+
function buildRole() {
|
|
6
|
+
let xml = (0, xmlbuilder2_1.create)()
|
|
7
|
+
.ele("Role")
|
|
8
|
+
.txt("You are a dataset transformer. Your goal is to read one or more existing JSONL datasets and produce a NEW JSONL dataset whose records conform exactly to the provided output schema.")
|
|
9
|
+
.up();
|
|
10
|
+
return xml.end({ prettyPrint: true, headless: true });
|
|
11
|
+
}
|
|
12
|
+
function buildGoal() {
|
|
13
|
+
let xml = (0, xmlbuilder2_1.create)()
|
|
14
|
+
.ele("Goal")
|
|
15
|
+
.txt("Transform the source dataset(s) (JSONL with {type:'row', data:{...}} per line) into a new dataset strictly matching the output schema. Save to output.jsonl in the dataset workstation. Each line must remain a single JSON object representing one record. You may need to combine, filter, or reshape data from multiple source datasets.")
|
|
16
|
+
.up();
|
|
17
|
+
return xml.end({ prettyPrint: true, headless: true });
|
|
18
|
+
}
|
|
19
|
+
function buildContextSection(context) {
|
|
20
|
+
let xml = (0, xmlbuilder2_1.create)()
|
|
21
|
+
.ele("Context")
|
|
22
|
+
.ele("DatasetId").txt(context.datasetId).up();
|
|
23
|
+
let sourcesXml = (0, xmlbuilder2_1.create)().ele("SourceDatasets");
|
|
24
|
+
for (const sourceId of context.sourceDatasetIds) {
|
|
25
|
+
sourcesXml = sourcesXml.ele("SourceDatasetId").txt(sourceId).up();
|
|
26
|
+
}
|
|
27
|
+
xml = xml.import(sourcesXml.first());
|
|
28
|
+
let sandboxXml = (0, xmlbuilder2_1.create)().ele("Sandbox");
|
|
29
|
+
for (const sourcePathInfo of context.sandboxConfig.sourcePaths) {
|
|
30
|
+
sandboxXml = sandboxXml.ele("SourceFile")
|
|
31
|
+
.ele("DatasetId").txt(sourcePathInfo.datasetId).up()
|
|
32
|
+
.ele("Path").txt(sourcePathInfo.path).up()
|
|
33
|
+
.up();
|
|
34
|
+
}
|
|
35
|
+
sandboxXml = sandboxXml.ele("OutputPath").txt(context.sandboxConfig.outputPath).up();
|
|
36
|
+
xml = xml.import(sandboxXml.first());
|
|
37
|
+
if (context.sourcePreviews && context.sourcePreviews.length > 0) {
|
|
38
|
+
let previewsXml = (0, xmlbuilder2_1.create)().ele("SourcePreviews");
|
|
39
|
+
for (const sourcePreviewInfo of context.sourcePreviews) {
|
|
40
|
+
const sp = sourcePreviewInfo.preview;
|
|
41
|
+
let px = (0, xmlbuilder2_1.create)().ele("SourcePreview")
|
|
42
|
+
.ele("DatasetId").txt(sourcePreviewInfo.datasetId).up()
|
|
43
|
+
.ele("TotalRows").txt(String(sp.totalRows)).up();
|
|
44
|
+
if (sp.metadata) {
|
|
45
|
+
const m = sp.metadata;
|
|
46
|
+
px = px.ele("Metadata")
|
|
47
|
+
.ele("Description").txt(m.description).up()
|
|
48
|
+
.ele("Script").txt(m.script).up()
|
|
49
|
+
.ele("Command").txt(m.command).up()
|
|
50
|
+
.ele("Stdout").txt(m.stdout).up();
|
|
51
|
+
if (m.stderr && m.stderr.trim().length > 0) {
|
|
52
|
+
px = px.ele("Stderr").txt(m.stderr).up();
|
|
53
|
+
}
|
|
54
|
+
px = px.up();
|
|
55
|
+
}
|
|
56
|
+
if (sp.head) {
|
|
57
|
+
const h = sp.head;
|
|
58
|
+
px = px.ele("Head")
|
|
59
|
+
.ele("Description").txt(h.description).up()
|
|
60
|
+
.ele("Script").txt(h.script).up()
|
|
61
|
+
.ele("Command").txt(h.command).up()
|
|
62
|
+
.ele("Stdout").txt(h.stdout).up();
|
|
63
|
+
if (h.stderr && h.stderr.trim().length > 0) {
|
|
64
|
+
px = px.ele("Stderr").txt(h.stderr).up();
|
|
65
|
+
}
|
|
66
|
+
px = px.up();
|
|
67
|
+
}
|
|
68
|
+
px = px.up();
|
|
69
|
+
previewsXml = previewsXml.import(px.first());
|
|
70
|
+
}
|
|
71
|
+
xml = xml.import(previewsXml.first());
|
|
72
|
+
}
|
|
73
|
+
if (Array.isArray(context.errors) && context.errors.length > 0) {
|
|
74
|
+
let ex = (0, xmlbuilder2_1.create)().ele("PreviousErrors");
|
|
75
|
+
for (const e of context.errors) {
|
|
76
|
+
ex = ex.ele("Error").txt(e).up();
|
|
77
|
+
}
|
|
78
|
+
xml = xml.import(ex.first());
|
|
79
|
+
}
|
|
80
|
+
xml = xml.up();
|
|
81
|
+
return xml.end({ prettyPrint: true, headless: true });
|
|
82
|
+
}
|
|
83
|
+
function buildOutputSchemaSection(context) {
|
|
84
|
+
let xml = (0, xmlbuilder2_1.create)()
|
|
85
|
+
.ele("OutputSchema")
|
|
86
|
+
.ele("JsonSchema").txt(JSON.stringify(context.outputSchema?.schema ?? context.outputSchema ?? {}, null, 2)).up()
|
|
87
|
+
.up();
|
|
88
|
+
return xml.end({ prettyPrint: true, headless: true });
|
|
89
|
+
}
|
|
90
|
+
function buildInstructions(context) {
|
|
91
|
+
const outputPath = context.sandboxConfig.outputPath;
|
|
92
|
+
const multipleSourcesNote = context.sourceDatasetIds.length > 1
|
|
93
|
+
? "You have multiple source datasets available. You may need to read, join, filter, or combine data from them to produce the output."
|
|
94
|
+
: "";
|
|
95
|
+
let xml = (0, xmlbuilder2_1.create)()
|
|
96
|
+
.ele("Instructions")
|
|
97
|
+
.ele("Workflow")
|
|
98
|
+
.ele("Step", { number: "1", name: "Inspect Source" })
|
|
99
|
+
.ele("Action").txt(`Review SourcePreviews to understand current record structures (data fields, shapes, edge cases). ${multipleSourcesNote}`).up()
|
|
100
|
+
.up()
|
|
101
|
+
.ele("Step", { number: "2", name: "Plan Mapping" })
|
|
102
|
+
.ele("Action").txt("Plan a deterministic mapping from source data fields to the output schema fields (normalize names, types, and formats).").up()
|
|
103
|
+
.ele("Note").txt("If fields are missing, set defaults; if types differ, coerce consistently. When working with multiple sources, decide how to combine or relate them.").up()
|
|
104
|
+
.up()
|
|
105
|
+
.ele("Step", { number: "3", name: "Transform" })
|
|
106
|
+
.ele("Action").txt("Use executeCommand to run a Python script that reads source JSONL file(s) and writes transformed records to output.jsonl. Keep line-per-record JSON objects with { 'type': 'row', 'data': { ... } }.").up()
|
|
107
|
+
.ele("Requirement").txt(`Write file to: ${outputPath}`).up()
|
|
108
|
+
.ele("Requirement").txt("Do not print large data to stdout; only progress and summaries.").up()
|
|
109
|
+
.up()
|
|
110
|
+
.ele("Step", { number: "4", name: "Validate and Complete" })
|
|
111
|
+
.ele("Action").txt("Call completeDataset to validate against the output schema and mark as completed.").up()
|
|
112
|
+
.up()
|
|
113
|
+
.up()
|
|
114
|
+
.ele("Rules")
|
|
115
|
+
.ele("Rule").txt("Output must strictly match the output schema for each record in data.").up()
|
|
116
|
+
.ele("Rule").txt("Each line in output.jsonl must be a standalone JSON object with {type:'row', data:{...}}.").up()
|
|
117
|
+
.ele("Rule").txt("Do not include headers, summaries, or metadata as records.").up()
|
|
118
|
+
.ele("Rule").txt("Be robust to malformed lines in source: skip or sanitize, but do not crash.").up()
|
|
119
|
+
.up()
|
|
120
|
+
.ele("CurrentTask").txt("Transform source dataset(s) to match OutputSchema and write output.jsonl, then complete.").up()
|
|
121
|
+
.up();
|
|
122
|
+
return xml.end({ prettyPrint: true, headless: true });
|
|
123
|
+
}
|
|
124
|
+
function buildTransformDatasetPrompt(context) {
|
|
125
|
+
const sections = [];
|
|
126
|
+
sections.push(buildRole());
|
|
127
|
+
sections.push("");
|
|
128
|
+
sections.push(buildGoal());
|
|
129
|
+
sections.push("");
|
|
130
|
+
sections.push(buildContextSection(context));
|
|
131
|
+
sections.push("");
|
|
132
|
+
sections.push(buildOutputSchemaSection(context));
|
|
133
|
+
sections.push("");
|
|
134
|
+
sections.push(buildInstructions(context));
|
|
135
|
+
return sections.join("\n");
|
|
136
|
+
}
|
|
137
|
+
//# sourceMappingURL=prompts.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"prompts.js","sourceRoot":"","sources":["../../src/transform/prompts.ts"],"names":[],"mappings":";;AAyKA,kEAYC;AArLD,6CAAoC;AAiCpC,SAAS,SAAS;IACd,IAAI,GAAG,GAAG,IAAA,oBAAM,GAAE;SACb,GAAG,CAAC,MAAM,CAAC;SACX,GAAG,CAAC,sLAAsL,CAAC;SAC3L,EAAE,EAAE,CAAA;IAET,OAAO,GAAG,CAAC,GAAG,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAA;AACzD,CAAC;AAED,SAAS,SAAS;IACd,IAAI,GAAG,GAAG,IAAA,oBAAM,GAAE;SACb,GAAG,CAAC,MAAM,CAAC;SACX,GAAG,CAAC,6UAA6U,CAAC;SAClV,EAAE,EAAE,CAAA;IAET,OAAO,GAAG,CAAC,GAAG,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAA;AACzD,CAAC;AAED,SAAS,mBAAmB,CAAC,OAA+B;IACxD,IAAI,GAAG,GAAG,IAAA,oBAAM,GAAE;SACb,GAAG,CAAC,SAAS,CAAC;SACd,GAAG,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAA;IAEjD,IAAI,UAAU,GAAG,IAAA,oBAAM,GAAE,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAA;IAC/C,KAAK,MAAM,QAAQ,IAAI,OAAO,CAAC,gBAAgB,EAAE,CAAC;QAC9C,UAAU,GAAG,UAAU,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAA;IACrE,CAAC;IACD,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC,CAAA;IAEpC,IAAI,UAAU,GAAG,IAAA,oBAAM,GAAE,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;IACxC,KAAK,MAAM,cAAc,IAAI,OAAO,CAAC,aAAa,CAAC,WAAW,EAAE,CAAC;QAC7D,UAAU,GAAG,UAAU,CAAC,GAAG,CAAC,YAAY,CAAC;aACpC,GAAG,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE;aACnD,GAAG,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE;aACzC,EAAE,EAAE,CAAA;IACb,CAAC;IACD,UAAU,GAAG,UAAU,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC,EAAE,EAAE,CAAA;IACpF,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC,CAAA;IAEpC,IAAI,OAAO,CAAC,cAAc,IAAI,OAAO,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9D,IAAI,WAAW,GAAG,IAAA,oBAAM,GAAE,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAA;QAChD,KAAK,MAAM,iBAAiB,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;YACrD,MAAM,EAAE,GAAG,iBAAiB,CAAC,OAAO,CAAA;YACpC,IAAI,EAAE,GAAG,IAAA,oBAAM,GAAE,CAAC,GAAG,CAAC,eAAe,CAAC;iBACjC,GAAG,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE;iBACtD,GAAG,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAA;YAEpD,IAAI,EAAE,CAAC,QAAQ,EAAE,CAAC;gBACd,MAAM,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAA;gBACrB,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC;qBAClB,GAAG,CAAC,aAAa,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,EAAE,EAAE;qBAC1C,GAAG,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE;qBAChC,GAAG,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE;qBAClC,GAAG,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,CAAA;gBACrC,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACzC,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,CAAA;gBAC5C,CAAC;gBACD,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE,CAAA;YAChB,CAAC;YAED,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;gBACV,MAAM,CAAC,GAAG,EAAE,CAAC,IAAI,CAAA;gBACjB,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC;qBACd,GAAG,CAAC,aAAa,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,EAAE,EAAE;qBAC1C,GAAG,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE;qBAChC,GAAG,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE;qBAClC,GAAG,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,CAAA;gBACrC,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACzC,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,CAAA;gBAC5C,CAAC;gBACD,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE,CAAA;YAChB,CAAC;YAED,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE,CAAA;YACZ,WAAW,GAAG,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,CAAA;QAChD,CAAC;QACD,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC,CAAA;IACzC,CAAC;IAED,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7D,IAAI,EAAE,GAAG,IAAA,oBAAM,GAAE,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAA;QACvC,KAAK,MAAM,CAAC,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YAC7B,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAA;QACpC,CAAC;QACD,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,CAAA;IAChC,CAAC;IAED,GAAG,GAAG,GAAG,CAAC,EAAE,EAAE,CAAA;IACd,OAAO,GAAG,CAAC,GAAG,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAA;AACzD,CAAC;AAED,SAAS,wBAAwB,CAAC,OAA+B;IAC7D,IAAI,GAAG,GAAG,IAAA,oBAAM,GAAE;SACb,GAAG,CAAC,cAAc,CAAC;SACnB,GAAG,CAAC,YAAY,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,YAAY,EAAE,MAAM,IAAI,OAAO,CAAC,YAAY,IAAI,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE;SAC/G,EAAE,EAAE,CAAA;IACT,OAAO,GAAG,CAAC,GAAG,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAA;AACzD,CAAC;AAED,SAAS,iBAAiB,CAAC,OAA+B;IACtD,MAAM,UAAU,GAAG,OAAO,CAAC,aAAa,CAAC,UAAU,CAAA;IACnD,MAAM,mBAAmB,GAAG,OAAO,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC;QAC3D,CAAC,CAAC,mIAAmI;QACrI,CAAC,CAAC,EAAE,CAAA;IAER,IAAI,GAAG,GAAG,IAAA,oBAAM,GAAE;SACb,GAAG,CAAC,cAAc,CAAC;SACnB,GAAG,CAAC,UAAU,CAAC;SACf,GAAG,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,gBAAgB,EAAE,CAAC;SACpD,GAAG,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,oGAAoG,mBAAmB,EAAE,CAAC,CAAC,EAAE,EAAE;SACjJ,EAAE,EAAE;SACJ,GAAG,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,cAAc,EAAE,CAAC;SAClD,GAAG,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,yHAAyH,CAAC,CAAC,EAAE,EAAE;SACjJ,GAAG,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,sJAAsJ,CAAC,CAAC,EAAE,EAAE;SAC5K,EAAE,EAAE;SACJ,GAAG,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC;SAC/C,GAAG,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,sMAAsM,CAAC,CAAC,EAAE,EAAE;SAC9N,GAAG,CAAC,aAAa,CAAC,CAAC,GAAG,CAAC,kBAAkB,UAAU,EAAE,CAAC,CAAC,EAAE,EAAE;SAC3D,GAAG,CAAC,aAAa,CAAC,CAAC,GAAG,CAAC,iEAAiE,CAAC,CAAC,EAAE,EAAE;SAC9F,EAAE,EAAE;SACJ,GAAG,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,uBAAuB,EAAE,CAAC;SAC3D,GAAG,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,mFAAmF,CAAC,CAAC,EAAE,EAAE;SAC3G,EAAE,EAAE;SACJ,EAAE,EAAE;SACJ,GAAG,CAAC,OAAO,CAAC;SACZ,GAAG,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,uEAAuE,CAAC,CAAC,EAAE,EAAE;SAC7F,GAAG,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,2FAA2F,CAAC,CAAC,EAAE,EAAE;SACjH,GAAG,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,4DAA4D,CAAC,CAAC,EAAE,EAAE;SAClF,GAAG,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,6EAA6E,CAAC,CAAC,EAAE,EAAE;SACnG,EAAE,EAAE;SACJ,GAAG,CAAC,aAAa,CAAC,CAAC,GAAG,CAAC,0FAA0F,CAAC,CAAC,EAAE,EAAE;SACvH,EAAE,EAAE,CAAA;IAET,OAAO,GAAG,CAAC,GAAG,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAA;AACzD,CAAC;AAED,SAAgB,2BAA2B,CAAC,OAA+B;IACvE,MAAM,QAAQ,GAAa,EAAE,CAAA;IAC7B,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,CAAA;IAC1B,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IACjB,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,CAAA;IAC1B,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IACjB,QAAQ,CAAC,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC,CAAA;IAC3C,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IACjB,QAAQ,CAAC,IAAI,CAAC,wBAAwB,CAAC,OAAO,CAAC,CAAC,CAAA;IAChD,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IACjB,QAAQ,CAAC,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAA;IACzC,OAAO,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AAC9B,CAAC"}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import { AgentOptions } from "@ekairos/story";
|
|
2
|
+
import { Sandbox } from "@vercel/sandbox";
|
|
3
|
+
import { DatasetService } from "../service";
|
|
4
|
+
import { TransformSourcePreviewContext } from "./filepreview";
|
|
5
|
+
export type TransformDatasetContext = {
|
|
6
|
+
datasetId: string;
|
|
7
|
+
sourceDatasetIds: string[];
|
|
8
|
+
outputSchema: any;
|
|
9
|
+
sandboxConfig: {
|
|
10
|
+
sourcePaths: Array<{
|
|
11
|
+
datasetId: string;
|
|
12
|
+
path: string;
|
|
13
|
+
}>;
|
|
14
|
+
outputPath: string;
|
|
15
|
+
};
|
|
16
|
+
sourcePreviews?: Array<{
|
|
17
|
+
datasetId: string;
|
|
18
|
+
preview: TransformSourcePreviewContext;
|
|
19
|
+
}>;
|
|
20
|
+
errors: string[];
|
|
21
|
+
iterationCount: number;
|
|
22
|
+
instructions?: string;
|
|
23
|
+
};
|
|
24
|
+
export type TransformDatasetAgentOptions = {
|
|
25
|
+
sourceDatasetIds: string[];
|
|
26
|
+
outputSchema: any;
|
|
27
|
+
sandbox: Sandbox;
|
|
28
|
+
service: DatasetService;
|
|
29
|
+
instructions?: string;
|
|
30
|
+
} & AgentOptions;
|
|
31
|
+
export type TransformDatasetResult = {
|
|
32
|
+
id: string;
|
|
33
|
+
status?: string;
|
|
34
|
+
title?: string;
|
|
35
|
+
schema?: any;
|
|
36
|
+
analysis?: any;
|
|
37
|
+
calculatedTotalRows?: number;
|
|
38
|
+
actualGeneratedRowCount?: number;
|
|
39
|
+
createdAt?: number;
|
|
40
|
+
updatedAt?: number;
|
|
41
|
+
};
|
|
42
|
+
export declare class TransformDatasetAgent {
|
|
43
|
+
private sourceDatasetIds;
|
|
44
|
+
private outputSchema;
|
|
45
|
+
private sandbox;
|
|
46
|
+
private service;
|
|
47
|
+
private agentService;
|
|
48
|
+
private instructions?;
|
|
49
|
+
constructor(params: {
|
|
50
|
+
sourceDatasetIds: string | string[];
|
|
51
|
+
outputSchema: any;
|
|
52
|
+
sandbox: Sandbox;
|
|
53
|
+
instructions?: string;
|
|
54
|
+
});
|
|
55
|
+
getDataset(): Promise<TransformDatasetResult>;
|
|
56
|
+
followUp(datasetId: string, feedback: string): Promise<TransformDatasetResult>;
|
|
57
|
+
}
|
|
58
|
+
//# sourceMappingURL=transform-dataset.agent.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"transform-dataset.agent.d.ts","sourceRoot":"","sources":["../../src/transform/transform-dataset.agent.ts"],"names":[],"mappings":"AAAA,OAAO,EAAS,YAAY,EAAoB,MAAM,gBAAgB,CAAA;AAGtE,OAAO,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAA;AAQzC,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAA;AAC3C,OAAO,EAAyB,6BAA6B,EAAE,MAAM,eAAe,CAAA;AAEpF,MAAM,MAAM,uBAAuB,GAAG;IAClC,SAAS,EAAE,MAAM,CAAA;IACjB,gBAAgB,EAAE,MAAM,EAAE,CAAA;IAC1B,YAAY,EAAE,GAAG,CAAA;IACjB,aAAa,EAAE;QACX,WAAW,EAAE,KAAK,CAAC;YAAE,SAAS,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;QACvD,UAAU,EAAE,MAAM,CAAA;KACrB,CAAA;IACD,cAAc,CAAC,EAAE,KAAK,CAAC;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,6BAA6B,CAAA;KAAE,CAAC,CAAA;IACrF,MAAM,EAAE,MAAM,EAAE,CAAA;IAChB,cAAc,EAAE,MAAM,CAAA;IACtB,YAAY,CAAC,EAAE,MAAM,CAAA;CACxB,CAAA;AAED,MAAM,MAAM,4BAA4B,GAAG;IACvC,gBAAgB,EAAE,MAAM,EAAE,CAAA;IAC1B,YAAY,EAAE,GAAG,CAAA;IACjB,OAAO,EAAE,OAAO,CAAA;IAChB,OAAO,EAAE,cAAc,CAAA;IACvB,YAAY,CAAC,EAAE,MAAM,CAAA;CACxB,GAAG,YAAY,CAAA;AAsMhB,MAAM,MAAM,sBAAsB,GAAG;IACjC,EAAE,EAAE,MAAM,CAAA;IACV,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,MAAM,CAAC,EAAE,GAAG,CAAA;IACZ,QAAQ,CAAC,EAAE,GAAG,CAAA;IACd,mBAAmB,CAAC,EAAE,MAAM,CAAA;IAC5B,uBAAuB,CAAC,EAAE,MAAM,CAAA;IAChC,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,SAAS,CAAC,EAAE,MAAM,CAAA;CACrB,CAAA;AAED,qBAAa,qBAAqB;IAC9B,OAAO,CAAC,gBAAgB,CAAU;IAClC,OAAO,CAAC,YAAY,CAAK;IACzB,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,OAAO,CAAgB;IAC/B,OAAO,CAAC,YAAY,CAAc;IAClC,OAAO,CAAC,YAAY,CAAC,CAAQ;gBAEjB,MAAM,EAAE;QAAE,gBAAgB,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;QAAC,YAAY,EAAE,GAAG,CAAC;QAAC,OAAO,EAAE,OAAO,CAAC;QAAC,YAAY,CAAC,EAAE,MAAM,CAAA;KAAE;IASjH,UAAU,IAAI,OAAO,CAAC,sBAAsB,CAAC;IA0D7C,QAAQ,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,sBAAsB,CAAC;CA0DvF"}
|