@ekairos/dataset 1.22.30-beta.development.0 → 1.22.32-beta.development.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/dist/dataset/steps.d.ts.map +1 -1
  2. package/dist/dataset/steps.js +22 -28
  3. package/dist/dataset/steps.js.map +1 -1
  4. package/dist/dataset.d.ts +67 -0
  5. package/dist/dataset.d.ts.map +1 -0
  6. package/dist/dataset.js +577 -0
  7. package/dist/dataset.js.map +1 -0
  8. package/dist/datasetFiles.d.ts +1 -0
  9. package/dist/datasetFiles.d.ts.map +1 -1
  10. package/dist/datasetFiles.js +21 -1
  11. package/dist/datasetFiles.js.map +1 -1
  12. package/dist/file/file-dataset.agent.d.ts +3 -1
  13. package/dist/file/file-dataset.agent.d.ts.map +1 -1
  14. package/dist/file/file-dataset.agent.js +22 -14
  15. package/dist/file/file-dataset.agent.js.map +1 -1
  16. package/dist/file/filepreview.js +1 -1
  17. package/dist/file/filepreview.js.map +1 -1
  18. package/dist/file/prompts.d.ts.map +1 -1
  19. package/dist/file/prompts.js +29 -12
  20. package/dist/file/prompts.js.map +1 -1
  21. package/dist/index.d.ts +3 -8
  22. package/dist/index.d.ts.map +1 -1
  23. package/dist/index.js +3 -8
  24. package/dist/index.js.map +1 -1
  25. package/dist/materializeDataset.tool.d.ts +78 -0
  26. package/dist/materializeDataset.tool.d.ts.map +1 -0
  27. package/dist/materializeDataset.tool.js +102 -0
  28. package/dist/materializeDataset.tool.js.map +1 -0
  29. package/dist/sandbox/steps.d.ts.map +1 -1
  30. package/dist/sandbox/steps.js +89 -0
  31. package/dist/sandbox/steps.js.map +1 -1
  32. package/dist/schema.d.ts +4 -1
  33. package/dist/schema.d.ts.map +1 -1
  34. package/dist/schema.js +4 -1
  35. package/dist/schema.js.map +1 -1
  36. package/dist/service.d.ts +14 -1
  37. package/dist/service.d.ts.map +1 -1
  38. package/dist/service.js +130 -10
  39. package/dist/service.js.map +1 -1
  40. package/dist/skill.d.ts +11 -0
  41. package/dist/skill.d.ts.map +1 -0
  42. package/dist/skill.js +53 -0
  43. package/dist/skill.js.map +1 -0
  44. package/dist/transform/transform-dataset.agent.d.ts +3 -0
  45. package/dist/transform/transform-dataset.agent.d.ts.map +1 -1
  46. package/dist/transform/transform-dataset.agent.js +12 -8
  47. package/dist/transform/transform-dataset.agent.js.map +1 -1
  48. package/package.json +12 -16
  49. package/skill/SKILL.md +83 -0
  50. package/skill/code/_runtime.mjs +156 -0
  51. package/skill/code/complete_dataset.mjs +104 -0
  52. package/skill/code/dataset_source_to_jsonl.mjs +51 -0
  53. package/skill/code/query_to_jsonl.mjs +22 -0
  54. package/skill/skill.toml +13 -0
@@ -1,3 +1,4 @@
1
+ import { type ContextReactor } from "@ekairos/events";
1
2
  import { TransformSourcePreviewContext } from "./filepreview";
2
3
  export type TransformDatasetContext = {
3
4
  datasetId: string;
@@ -25,6 +26,7 @@ export type TransformDatasetAgentParams = {
25
26
  datasetId?: string;
26
27
  model?: string;
27
28
  sandboxId?: string;
29
+ reactor?: ContextReactor<any, any>;
28
30
  };
29
31
  export type TransformDatasetResult = {
30
32
  id: string;
@@ -46,6 +48,7 @@ export declare function createTransformDatasetStory<Env extends {
46
48
  datasetId?: string;
47
49
  model?: string;
48
50
  sandboxId?: string;
51
+ reactor?: ContextReactor<any, any>;
49
52
  }): {
50
53
  datasetId: string;
51
54
  transform(env: Env, prompt?: string): Promise<{
@@ -1 +1 @@
1
- {"version":3,"file":"transform-dataset.agent.d.ts","sourceRoot":"","sources":["../../src/transform/transform-dataset.agent.ts"],"names":[],"mappings":"AAOA,OAAO,EAAyB,6BAA6B,EAAE,MAAM,eAAe,CAAA;AAIpF,MAAM,MAAM,uBAAuB,GAAG;IAClC,SAAS,EAAE,MAAM,CAAA;IACjB,gBAAgB,EAAE,MAAM,EAAE,CAAA;IAC1B,YAAY,EAAE,GAAG,CAAA;IACjB,aAAa,EAAE;QACX,WAAW,EAAE,KAAK,CAAC;YAAE,SAAS,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;QACvD,UAAU,EAAE,MAAM,CAAA;KACrB,CAAA;IACD,cAAc,CAAC,EAAE,KAAK,CAAC;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,6BAA6B,CAAA;KAAE,CAAC,CAAA;IACrF,MAAM,EAAE,MAAM,EAAE,CAAA;IAChB,cAAc,EAAE,MAAM,CAAA;IACtB,YAAY,CAAC,EAAE,MAAM,CAAA;CACxB,CAAA;AAED,MAAM,MAAM,2BAA2B,GAAG;IACtC,gBAAgB,EAAE,MAAM,EAAE,CAAA;IAC1B,YAAY,EAAE,GAAG,CAAA;IACjB,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,SAAS,CAAC,EAAE,MAAM,CAAA;CACrB,CAAA;AA4CD,MAAM,MAAM,sBAAsB,GAAG;IACjC,EAAE,EAAE,MAAM,CAAA;IACV,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,MAAM,CAAC,EAAE,GAAG,CAAA;IACZ,QAAQ,CAAC,EAAE,GAAG,CAAA;IACd,mBAAmB,CAAC,EAAE,MAAM,CAAA;IAC5B,uBAAuB,CAAC,EAAE,MAAM,CAAA;IAChC,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,SAAS,CAAC,EAAE,MAAM,CAAA;CACrB,CAAA;AA6GD,wBAAgB,2BAA2B,CAAC,GAAG,SAAS;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,EACrE,MAAM,EAAE;IACJ,gBAAgB,EAAE,MAAM,EAAE,CAAA;IAC1B,YAAY,EAAE,GAAG,CAAA;IACjB,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,SAAS,CAAC,EAAE,MAAM,CAAA;CACrB;;mBAawB,GAAG,WAAW,MAAM,GAAG,OAAO,CAAC;QAAE,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC;;EAiCjF"}
1
+ {"version":3,"file":"transform-dataset.agent.d.ts","sourceRoot":"","sources":["../../src/transform/transform-dataset.agent.ts"],"names":[],"mappings":"AAAA,OAAO,EAAoE,KAAK,cAAc,EAAE,MAAM,iBAAiB,CAAA;AAOvH,OAAO,EAAyB,6BAA6B,EAAE,MAAM,eAAe,CAAA;AAIpF,MAAM,MAAM,uBAAuB,GAAG;IAClC,SAAS,EAAE,MAAM,CAAA;IACjB,gBAAgB,EAAE,MAAM,EAAE,CAAA;IAC1B,YAAY,EAAE,GAAG,CAAA;IACjB,aAAa,EAAE;QACX,WAAW,EAAE,KAAK,CAAC;YAAE,SAAS,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;QACvD,UAAU,EAAE,MAAM,CAAA;KACrB,CAAA;IACD,cAAc,CAAC,EAAE,KAAK,CAAC;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,6BAA6B,CAAA;KAAE,CAAC,CAAA;IACrF,MAAM,EAAE,MAAM,EAAE,CAAA;IAChB,cAAc,EAAE,MAAM,CAAA;IACtB,YAAY,CAAC,EAAE,MAAM,CAAA;CACxB,CAAA;AAED,MAAM,MAAM,2BAA2B,GAAG;IACtC,gBAAgB,EAAE,MAAM,EAAE,CAAA;IAC1B,YAAY,EAAE,GAAG,CAAA;IACjB,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,OAAO,CAAC,EAAE,cAAc,CAAC,GAAG,EAAE,GAAG,CAAC,CAAA;CACrC,CAAA;AA4CD,MAAM,MAAM,sBAAsB,GAAG;IACjC,EAAE,EAAE,MAAM,CAAA;IACV,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,MAAM,CAAC,EAAE,GAAG,CAAA;IACZ,QAAQ,CAAC,EAAE,GAAG,CAAA;IACd,mBAAmB,CAAC,EAAE,MAAM,CAAA;IAC5B,uBAAuB,CAAC,EAAE,MAAM,CAAA;IAChC,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,SAAS,CAAC,EAAE,MAAM,CAAA;CACrB,CAAA;AAgHD,wBAAgB,2BAA2B,CAAC,GAAG,SAAS;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,EACrE,MAAM,EAAE;IACJ,gBAAgB,EAAE,MAAM,EAAE,CAAA;IAC1B,YAAY,EAAE,GAAG,CAAA;IACjB,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,OAAO,CAAC,EAAE,cAAc,CAAC,GAAG,EAAE,GAAG,CAAC,CAAA;CACrC;;mBAcwB,GAAG,WAAW,MAAM,GAAG,OAAO,CAAC;QAAE,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC;;EAiCjF"}
@@ -35,15 +35,13 @@ async function ensureSourcesInSandbox(env, sandboxId, datasetId, sourceDatasetId
35
35
  function createTransformDatasetStoryDefinition(params) {
36
36
  const datasetId = params.datasetId ?? (0, admin_1.id)();
37
37
  const model = params.model ?? "openai/gpt-5";
38
- const story = (0, events_1.createContext)("dataset.transform")
38
+ let storyBuilder = (0, events_1.createContext)("dataset.transform")
39
39
  .context(async (stored, env) => {
40
40
  const previous = stored?.content ?? {};
41
41
  const sandboxState = previous?.sandboxState ?? { initialized: false, sourcePaths: [] };
42
- const existingSandboxId = previous?.sandboxId ?? params.sandboxId ?? "";
43
- let sandboxId = existingSandboxId;
42
+ const sandboxId = previous?.sandboxId ?? params.sandboxId ?? "";
44
43
  if (!sandboxId) {
45
- const created = await (0, steps_2.createDatasetSandboxStep)({ env, runtime: "python3.13", timeoutMs: 10 * 60 * 1000 });
46
- sandboxId = created.sandboxId;
44
+ throw new Error("dataset_sandbox_required");
47
45
  }
48
46
  const { sourcePaths, outputPath } = await ensureSourcesInSandbox(env, sandboxId, datasetId, params.sourceDatasetIds, sandboxState);
49
47
  const sourcePreviews = [];
@@ -117,9 +115,14 @@ function createTransformDatasetStoryDefinition(params) {
117
115
  })
118
116
  .shouldContinue(({ reactionEvent }) => {
119
117
  return !(0, events_1.didToolExecute)(reactionEvent, "completeDataset");
120
- })
121
- .model(model)
122
- .build();
118
+ });
119
+ if (params.reactor) {
120
+ storyBuilder = storyBuilder.reactor(params.reactor);
121
+ }
122
+ else {
123
+ storyBuilder = storyBuilder.model(model);
124
+ }
125
+ const story = storyBuilder.build();
123
126
  return { datasetId, story };
124
127
  }
125
128
  function createTransformDatasetStory(params) {
@@ -130,6 +133,7 @@ function createTransformDatasetStory(params) {
130
133
  datasetId: params.datasetId,
131
134
  model: params.model,
132
135
  sandboxId: params.sandboxId,
136
+ reactor: params.reactor,
133
137
  });
134
138
  return {
135
139
  datasetId,
@@ -1 +1 @@
1
- {"version":3,"file":"transform-dataset.agent.js","sourceRoot":"","sources":["../../src/transform/transform-dataset.agent.ts"],"names":[],"mappings":";;AAmMA,kEAsDC;AAzPD,4CAAkG;AAClG,kEAAmE;AACnE,gEAAiE;AACjE,4DAA6D;AAC7D,uCAA+E;AAC/E,kDAA6E;AAC7E,4CAAqC;AACrC,+CAAoF;AACpF,4CAAsF;AACtF,4CAAuH;AA+BvH,KAAK,UAAU,sBAAsB,CACjC,GAAQ,EACR,SAAiB,EACjB,SAAiB,EACjB,gBAA0B,EAC1B,KAA4B;IAE5B,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC;QACpB,OAAO,EAAE,WAAW,EAAE,KAAK,CAAC,WAAW,EAAE,UAAU,EAAE,IAAA,mCAAoB,EAAC,SAAS,CAAC,EAAE,CAAA;IAC1F,CAAC;IAED,MAAM,WAAW,GAAG,IAAA,oCAAqB,EAAC,SAAS,CAAC,CAAA;IAEpD,MAAM,IAAA,oCAA4B,EAAC,EAAE,GAAG,EAAE,SAAS,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,IAAI,EAAE,WAAW,CAAC,EAAE,CAAC,CAAA;IAE/F,MAAM,WAAW,GAA+C,EAAE,CAAA;IAElE,KAAK,MAAM,eAAe,IAAI,gBAAgB,EAAE,CAAC;QAC7C,MAAM,UAAU,GAAG,GAAG,WAAW,WAAW,eAAe,QAAQ,CAAA;QAEnE,MAAM,MAAM,GAAG,MAAM,IAAA,kCAA0B,EAAC,EAAE,GAAG,EAAE,SAAS,EAAE,eAAe,EAAE,CAAC,CAAA;QACpF,MAAM,IAAA,oCAA4B,EAAC;YAC/B,GAAG;YACH,SAAS;YACT,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,CAAC,aAAa,EAAE,CAAC;SACrE,CAAC,CAAA;QAEF,WAAW,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,eAAe,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,CAAA;IACtE,CAAC;IAED,KAAK,CAAC,WAAW,GAAG,WAAW,CAAA;IAC/B,KAAK,CAAC,WAAW,GAAG,IAAI,CAAA;IAExB,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,IAAA,mCAAoB,EAAC,SAAS,CAAC,EAAE,CAAA;AACvE,CAAC;AAcD,SAAS,qCAAqC,CAC1C,MAAmC;IAEnC,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,IAAI,IAAA,UAAE,GAAE,CAAA;IAC1C,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,cAAc,CAAA;IAE5C,MAAM,KAAK,GAAG,IAAA,sBAAa,EAAM,mBAAmB,CAAC;SAChD,OAAO,CAAC,KAAK,EAAE,MAAW,EAAE,GAAQ,EAAE,EAAE;QACrC,MAAM,QAAQ,GAAI,MAAM,EAAE,OAAe,IAAI,EAAE,CAAA;QAC/C,MAAM,YAAY,GAA0B,QAAQ,EAAE,YAAY,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,WAAW,EAAE,EAAE,EAAE,CAAA;QAC7G,MAAM,iBAAiB,GAAW,QAAQ,EAAE,SAAS,IAAI,MAAM,CAAC,SAAS,IAAI,EAAE,CAAA;QAE/E,IAAI,SAAS,GAAG,iBAAiB,CAAA;QACjC,IAAI,CAAC,SAAS,EAAE,CAAC;YACb,MAAM,OAAO,GAAG,MAAM,IAAA,gCAAwB,EAAC,EAAE,GAAG,EAAE,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,EAAE,GAAG,EAAE,GAAG,IAAI,EAAE,CAAC,CAAA;YACzG,SAAS,GAAG,OAAO,CAAC,SAAS,CAAA;QACjC,CAAC;QAEL,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,GAAG,MAAM,sBAAsB,CACxD,GAAG,EACH,SAAS,EACb,SAAS,EACL,MAAM,CAAC,gBAAgB,EACvB,YAAY,CACnB,CAAA;QAED,MAAM,cAAc,GAAyE,EAAE,CAAA;QAC3F,KAAK,MAAM,EAAE,IAAI,WAAW,EAAE,CAAC;YAC/B,IAAI,CAAC;gBACG,MAAM,OAAO,GAAG,MAAM,IAAA,mCAAqB,EAAC,GAAG,EAAE,SAAS,EAAE,EAAE,CAAC,IAAI,EAAE,SAAS,CAAC,CAAA;gBAC/E,cAAc,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,EAAE,CAAC,SAAS,EAAE,OAAO,EAAE,CAAC,CAAA;YACjE,CAAC;YAAC,MAAM,CAAC;gBACL,WAAW;YACf,CAAC;QACL,CAAC;QAEG,wFAAwF;QACxF,MAAM,IAAA,+BAAuB,EAAC;YAC1B,GAAG;YACH,SAAS;YACT,MAAM,EAAE,MAAM,CAAC,YAAY;YAC3B,MAAM,EAAE,iBAAiB;SAC5B,CAAC,CAAA;QAEF,MAAM,aAAa,GAA2B;YAC9C,SAAS;YACL,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;YACzC,YAAY,EAAE,MAAM,CAAC,YAAY;YACrC,aAAa,EAAE,EAAE,WAAW,EAAE,UAAU,EAAE;YAC1C,cAAc,EAAE,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,SAAS;YACtE,MAAM,EAAE,EAAE;SACT,CAAA;QAED,MAAM,UAAU,GAAG,IAAA,qCAA2B,EAAC,aAAa,CAAC,CAAA;QAC7D,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;QACjE,MAAM,MAAM,GAAG,gBAAgB;YAC3B,CAAC,CAAC;gBACE,sBAAsB;gBACtB,2IAA2I;gBAC3I,EAAE;gBACF,gBAAgB;gBAChB,EAAE;gBACF,UAAU;aACb,CAAC,IAAI,CAAC,IAAI,CAAC;YACZ,CAAC,CAAC,UAAU,CAAA;QAEhB,OAAO;YACH,GAAG,QAAQ;YACX,SAAS;YACT,SAAS;YACT,YAAY;YACZ,MAAM;YACN,aAAa,EAAE,EAAE,WAAW,EAAE,UAAU,EAAE;SAC7C,CAAA;IACL,CAAC,CAAC;SACD,SAAS,CAAC,KAAK,EAAE,MAAW,EAAE,EAAE;QAC7B,OAAO,MAAM,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,IAAI,EAAE,CAAC,CAAA;IAChD,CAAC,CAAC;SACD,OAAO,CAAC,KAAK,EAAE,MAAW,EAAE,GAAQ,EAAE,EAAE;QACrC,MAAM,SAAS,GAAI,MAAM,EAAE,OAAO,EAAE,SAAoB,IAAI,MAAM,CAAC,SAAS,IAAI,EAAE,CAAA;QAClF,OAAO;YACP,cAAc,EAAE,IAAA,8CAAwB,EAAC;gBACjC,SAAS;gBACT,SAAS;gBACT,GAAG;aACV,CAAC;YACF,eAAe,EAAE,IAAA,gDAAyB,EAAC;gBACnC,SAAS;gBACT,SAAS;gBACT,GAAG;aACV,CAAC;YACF,YAAY,EAAE,IAAA,0CAAsB,EAAC;gBAC7B,SAAS;gBACT,SAAS;gBACT,GAAG;aACN,CAAC;SACM,CAAA;IACZ,CAAC,CAAC;SACD,cAAc,CAAC,CAAC,EAAE,aAAa,EAA0B,EAAE,EAAE;QAC1D,OAAO,CAAC,IAAA,uBAAc,EAAC,aAAoB,EAAE,iBAAiB,CAAC,CAAA;IACnE,CAAC,CAAC;SACL,KAAK,CAAC,KAAK,CAAC;SACZ,KAAK,EAAE,CAAA;IAEZ,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,CAAA;AAC/B,CAAC;AAED,SAAgB,2BAA2B,CACvC,MAOC;IAED,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,GAAG,qCAAqC,CAAM;QACpE,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;QACzC,YAAY,EAAE,MAAM,CAAC,YAAY;QACjC,YAAY,EAAE,MAAM,CAAC,YAAY;QACjC,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,SAAS,EAAE,MAAM,CAAC,SAAS;KAC9B,CAAC,CAAA;IAEF,OAAO;QACH,SAAS;QACT,KAAK,CAAC,SAAS,CAAC,GAAQ,EAAE,MAAe;YACrC,MAAM,gBAAgB,GAClB,MAAM,CAAC,gBAAgB,CAAC,MAAM,KAAK,CAAC;gBAChC,CAAC,CAAC,oBAAoB;gBACtB,CAAC,CAAC,GAAG,MAAM,CAAC,gBAAgB,CAAC,MAAM,kBAAkB,CAAA;YAEjE,MAAM,YAAY,GAAG;gBACjB,EAAE,EAAE,IAAA,UAAE,GAAE;gBACR,IAAI,EAAE,6BAAoB;gBAC1B,OAAO,EAAE,oBAAW;gBACpB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACnC,OAAO,EAAE;oBACD,KAAK,EAAE;wBACH;4BACI,IAAI,EAAE,MAAM;4BACZ,IAAI,EACA,MAAM;gCACN,aAAa,gBAAgB,yDAAyD;yBAC7F;qBACJ;iBACR;aACG,CAAA;YAER,MAAM,KAAK,CAAC,KAAK,CAAC,YAAY,EAAE;gBACxB,GAAG;gBACH,OAAO,EAAE,EAAE,GAAG,EAAE,WAAW,SAAS,EAAE,EAAE;gBAC5C,OAAO,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,aAAa,EAAE,EAAE,EAAE,aAAa,EAAE,CAAC,EAAE;aACxG,CAAC,CAAA;YAEE,OAAO,EAAE,SAAS,EAAE,CAAA;QACxB,CAAC;QACD,KAAK;KACR,CAAA;AACL,CAAC"}
1
+ {"version":3,"file":"transform-dataset.agent.js","sourceRoot":"","sources":["../../src/transform/transform-dataset.agent.ts"],"names":[],"mappings":";;AAuMA,kEAwDC;AA/PD,4CAAuH;AACvH,kEAAmE;AACnE,gEAAiE;AACjE,4DAA6D;AAC7D,uCAA+E;AAC/E,kDAA6E;AAC7E,4CAAqC;AACrC,+CAAoF;AACpF,4CAAsF;AACtF,4CAA6F;AAgC7F,KAAK,UAAU,sBAAsB,CACjC,GAAQ,EACR,SAAiB,EACjB,SAAiB,EACjB,gBAA0B,EAC1B,KAA4B;IAE5B,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC;QACpB,OAAO,EAAE,WAAW,EAAE,KAAK,CAAC,WAAW,EAAE,UAAU,EAAE,IAAA,mCAAoB,EAAC,SAAS,CAAC,EAAE,CAAA;IAC1F,CAAC;IAED,MAAM,WAAW,GAAG,IAAA,oCAAqB,EAAC,SAAS,CAAC,CAAA;IAEpD,MAAM,IAAA,oCAA4B,EAAC,EAAE,GAAG,EAAE,SAAS,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,IAAI,EAAE,WAAW,CAAC,EAAE,CAAC,CAAA;IAE/F,MAAM,WAAW,GAA+C,EAAE,CAAA;IAElE,KAAK,MAAM,eAAe,IAAI,gBAAgB,EAAE,CAAC;QAC7C,MAAM,UAAU,GAAG,GAAG,WAAW,WAAW,eAAe,QAAQ,CAAA;QAEnE,MAAM,MAAM,GAAG,MAAM,IAAA,kCAA0B,EAAC,EAAE,GAAG,EAAE,SAAS,EAAE,eAAe,EAAE,CAAC,CAAA;QACpF,MAAM,IAAA,oCAA4B,EAAC;YAC/B,GAAG;YACH,SAAS;YACT,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,CAAC,aAAa,EAAE,CAAC;SACrE,CAAC,CAAA;QAEF,WAAW,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,eAAe,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,CAAA;IACtE,CAAC;IAED,KAAK,CAAC,WAAW,GAAG,WAAW,CAAA;IAC/B,KAAK,CAAC,WAAW,GAAG,IAAI,CAAA;IAExB,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,IAAA,mCAAoB,EAAC,SAAS,CAAC,EAAE,CAAA;AACvE,CAAC;AAcD,SAAS,qCAAqC,CAC1C,MAAmC;IAEnC,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,IAAI,IAAA,UAAE,GAAE,CAAA;IAC1C,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,cAAc,CAAA;IAE5C,IAAI,YAAY,GAAG,IAAA,sBAAa,EAAM,mBAAmB,CAAC;SACrD,OAAO,CAAC,KAAK,EAAE,MAAW,EAAE,GAAQ,EAAE,EAAE;QACrC,MAAM,QAAQ,GAAI,MAAM,EAAE,OAAe,IAAI,EAAE,CAAA;QAC/C,MAAM,YAAY,GAA0B,QAAQ,EAAE,YAAY,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,WAAW,EAAE,EAAE,EAAE,CAAA;QAC7G,MAAM,SAAS,GAAW,QAAQ,EAAE,SAAS,IAAI,MAAM,CAAC,SAAS,IAAI,EAAE,CAAA;QACvE,IAAI,CAAC,SAAS,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAA;QAC/C,CAAC;QAEL,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,GAAG,MAAM,sBAAsB,CACxD,GAAG,EACH,SAAS,EACb,SAAS,EACL,MAAM,CAAC,gBAAgB,EACvB,YAAY,CACnB,CAAA;QAED,MAAM,cAAc,GAAyE,EAAE,CAAA;QAC3F,KAAK,MAAM,EAAE,IAAI,WAAW,EAAE,CAAC;YAC/B,IAAI,CAAC;gBACG,MAAM,OAAO,GAAG,MAAM,IAAA,mCAAqB,EAAC,GAAG,EAAE,SAAS,EAAE,EAAE,CAAC,IAAI,EAAE,SAAS,CAAC,CAAA;gBAC/E,cAAc,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,EAAE,CAAC,SAAS,EAAE,OAAO,EAAE,CAAC,CAAA;YACjE,CAAC;YAAC,MAAM,CAAC;gBACL,WAAW;YACf,CAAC;QACL,CAAC;QAEG,wFAAwF;QACxF,MAAM,IAAA,+BAAuB,EAAC;YAC1B,GAAG;YACH,SAAS;YACT,MAAM,EAAE,MAAM,CAAC,YAAY;YAC3B,MAAM,EAAE,iBAAiB;SAC5B,CAAC,CAAA;QAEF,MAAM,aAAa,GAA2B;YAC9C,SAAS;YACL,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;YACzC,YAAY,EAAE,MAAM,CAAC,YAAY;YACrC,aAAa,EAAE,EAAE,WAAW,EAAE,UAAU,EAAE;YAC1C,cAAc,EAAE,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,SAAS;YACtE,MAAM,EAAE,EAAE;SACT,CAAA;QAED,MAAM,UAAU,GAAG,IAAA,qCAA2B,EAAC,aAAa,CAAC,CAAA;QAC7D,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;QACjE,MAAM,MAAM,GAAG,gBAAgB;YAC3B,CAAC,CAAC;gBACE,sBAAsB;gBACtB,2IAA2I;gBAC3I,EAAE;gBACF,gBAAgB;gBAChB,EAAE;gBACF,UAAU;aACb,CAAC,IAAI,CAAC,IAAI,CAAC;YACZ,CAAC,CAAC,UAAU,CAAA;QAEhB,OAAO;YACH,GAAG,QAAQ;YACX,SAAS;YACT,SAAS;YACT,YAAY;YACZ,MAAM;YACN,aAAa,EAAE,EAAE,WAAW,EAAE,UAAU,EAAE;SAC7C,CAAA;IACL,CAAC,CAAC;SACD,SAAS,CAAC,KAAK,EAAE,MAAW,EAAE,EAAE;QAC7B,OAAO,MAAM,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,IAAI,EAAE,CAAC,CAAA;IAChD,CAAC,CAAC;SACD,OAAO,CAAC,KAAK,EAAE,MAAW,EAAE,GAAQ,EAAE,EAAE;QACrC,MAAM,SAAS,GAAI,MAAM,EAAE,OAAO,EAAE,SAAoB,IAAI,MAAM,CAAC,SAAS,IAAI,EAAE,CAAA;QAClF,OAAO;YACP,cAAc,EAAE,IAAA,8CAAwB,EAAC;gBACjC,SAAS;gBACT,SAAS;gBACT,GAAG;aACV,CAAC;YACF,eAAe,EAAE,IAAA,gDAAyB,EAAC;gBACnC,SAAS;gBACT,SAAS;gBACT,GAAG;aACV,CAAC;YACF,YAAY,EAAE,IAAA,0CAAsB,EAAC;gBAC7B,SAAS;gBACT,SAAS;gBACT,GAAG;aACN,CAAC;SACM,CAAA;IACZ,CAAC,CAAC;SACD,cAAc,CAAC,CAAC,EAAE,aAAa,EAA0B,EAAE,EAAE;QAC1D,OAAO,CAAC,IAAA,uBAAc,EAAC,aAAoB,EAAE,iBAAiB,CAAC,CAAA;IACnE,CAAC,CAAC,CAAA;IAEV,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QACjB,YAAY,GAAG,YAAY,CAAC,OAAO,CAAC,MAAM,CAAC,OAAc,CAAC,CAAA;IAC9D,CAAC;SAAM,CAAC;QACJ,YAAY,GAAG,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;IAC5C,CAAC;IAED,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,EAAE,CAAA;IAElC,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,CAAA;AAC/B,CAAC;AAED,SAAgB,2BAA2B,CACvC,MAQC;IAED,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,GAAG,qCAAqC,CAAM;QACpE,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;QACzC,YAAY,EAAE,MAAM,CAAC,YAAY;QACjC,YAAY,EAAE,MAAM,CAAC,YAAY;QACjC,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,OAAO,EAAE,MAAM,CAAC,OAAO;KAC1B,CAAC,CAAA;IAEF,OAAO;QACH,SAAS;QACT,KAAK,CAAC,SAAS,CAAC,GAAQ,EAAE,MAAe;YACrC,MAAM,gBAAgB,GAClB,MAAM,CAAC,gBAAgB,CAAC,MAAM,KAAK,CAAC;gBAChC,CAAC,CAAC,oBAAoB;gBACtB,CAAC,CAAC,GAAG,MAAM,CAAC,gBAAgB,CAAC,MAAM,kBAAkB,CAAA;YAEjE,MAAM,YAAY,GAAG;gBACjB,EAAE,EAAE,IAAA,UAAE,GAAE;gBACR,IAAI,EAAE,6BAAoB;gBAC1B,OAAO,EAAE,oBAAW;gBACpB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACnC,OAAO,EAAE;oBACD,KAAK,EAAE;wBACH;4BACI,IAAI,EAAE,MAAM;4BACZ,IAAI,EACA,MAAM;gCACN,aAAa,gBAAgB,yDAAyD;yBAC7F;qBACJ;iBACR;aACG,CAAA;YAER,MAAM,KAAK,CAAC,KAAK,CAAC,YAAY,EAAE;gBACxB,GAAG;gBACH,OAAO,EAAE,EAAE,GAAG,EAAE,WAAW,SAAS,EAAE,EAAE;gBAC5C,OAAO,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,aAAa,EAAE,EAAE,EAAE,aAAa,EAAE,CAAC,EAAE;aACxG,CAAC,CAAA;YAEE,OAAO,EAAE,SAAS,EAAE,CAAA;QACxB,CAAC;QACD,KAAK;KACR,CAAA;AACL,CAAC"}
package/package.json CHANGED
@@ -1,13 +1,14 @@
1
1
  {
2
2
  "name": "@ekairos/dataset",
3
- "version": "1.22.30-beta.development.0",
3
+ "version": "1.22.32-beta.development.0",
4
4
  "description": "Pulzar Dataset Tools",
5
5
  "type": "commonjs",
6
6
  "main": "dist/index.js",
7
7
  "types": "dist/index.d.ts",
8
8
  "files": [
9
9
  "dist",
10
- "file/scripts"
10
+ "file/scripts",
11
+ "skill"
11
12
  ],
12
13
  "publishConfig": {
13
14
  "access": "public"
@@ -24,16 +25,6 @@
24
25
  "require": "./dist/index.js",
25
26
  "default": "./dist/index.js"
26
27
  },
27
- "./query": {
28
- "types": "./dist/query/index.d.ts",
29
- "require": "./dist/query/index.js",
30
- "default": "./dist/query/index.js"
31
- },
32
- "./transform": {
33
- "types": "./dist/transform/index.d.ts",
34
- "require": "./dist/transform/index.js",
35
- "default": "./dist/transform/index.js"
36
- },
37
28
  "./file/scripts": {
38
29
  "default": "./dist/file/scripts"
39
30
  },
@@ -46,12 +37,15 @@
46
37
  "dev": "tsc -p tsconfig.json --watch",
47
38
  "clean": "node -e \"require('fs').rmSync('dist', {recursive:true, force:true})\"",
48
39
  "typecheck": "tsc --noEmit",
49
- "test": "vitest run -c vitest.config.mts"
40
+ "test": "vitest run -c vitest.config.mts",
41
+ "test:direct": "vitest run -c vitest.config.mts src/tests/dataset.builder.instant.test.ts",
42
+ "test:tool": "vitest run -c vitest.config.mts src/tests/materializeDataset.tool.instant.test.ts",
43
+ "test:ai-sdk:instant": "vitest run -c vitest.codex.config.mts src/tests/materializeDataset.ai-sdk.instant.test.ts"
50
44
  },
51
45
  "dependencies": {
52
- "@ekairos/domain": "^1.22.30-beta.development.0",
53
- "@ekairos/sandbox": "^1.22.30-beta.development.0",
54
- "@ekairos/events": "^1.22.30-beta.development.0",
46
+ "@ekairos/domain": "^1.22.32-beta.development.0",
47
+ "@ekairos/sandbox": "^1.22.32-beta.development.0",
48
+ "@ekairos/events": "^1.22.32-beta.development.0",
55
49
  "@instantdb/admin": "0.22.126",
56
50
  "@instantdb/core": "0.22.126",
57
51
  "ai": "^5.0.44",
@@ -60,6 +54,8 @@
60
54
  "zod": "^4.1.8"
61
55
  },
62
56
  "devDependencies": {
57
+ "@ai-sdk/openai": "^2.0.42",
58
+ "@ekairos/openai-reactor": "workspace:*",
63
59
  "@ekairos/tsconfig": "workspace:*",
64
60
  "@types/node": "^24.5.0",
65
61
  "dotenv": "^17.2.3",
package/skill/SKILL.md ADDED
@@ -0,0 +1,83 @@
1
+ ---
2
+ name: dataset
3
+ description: Materialize, transform, and persist datasets using the Ekairos runtime manifest and InstantDB HTTP API.
4
+ ---
5
+
6
+ # dataset
7
+
8
+ Use this skill when a task requires turning queries, local files, or existing datasets into a persisted dataset result.
9
+
10
+ The skill assumes:
11
+ - the current environment provides `EKAIROS_RUNTIME_MANIFEST_PATH`, or the manifest is at `.ekairos/runtime.json`
12
+ - the runtime manifest contains `appId`, domain context, and optionally a scoped token
13
+ - when no token is present in the manifest, network/header injection will provide `as-token`
14
+
15
+ ## What this skill provides
16
+
17
+ - `code/query_to_jsonl.mjs`
18
+ - Runs an Instant query and writes normalized dataset rows to a local JSONL file.
19
+ - `code/dataset_source_to_jsonl.mjs`
20
+ - Downloads an existing dataset output file to a local JSONL file.
21
+ - `code/complete_dataset.mjs`
22
+ - Uploads a local JSONL output file to Instant storage and upserts the target dataset metadata.
23
+
24
+ ## Usage pattern
25
+
26
+ 1. Create a small JSON input file for the command you want to run.
27
+ 2. Run the appropriate `node code/*.mjs <input.json>` command.
28
+ 3. For transformations:
29
+ - prepare one or more source JSONL files,
30
+ - use your own shell/python/node transform to create the final output JSONL,
31
+ - run `complete_dataset.mjs` to persist the dataset.
32
+
33
+ ## Input formats
34
+
35
+ ### `query_to_jsonl.mjs`
36
+
37
+ ```json
38
+ {
39
+ "query": { "items": {} },
40
+ "outputPath": "./work/source.jsonl",
41
+ "manifestPath": "./.ekairos/runtime.json"
42
+ }
43
+ ```
44
+
45
+ ### `dataset_source_to_jsonl.mjs`
46
+
47
+ ```json
48
+ {
49
+ "datasetId": "source_dataset_v1",
50
+ "outputPath": "./work/source_dataset.jsonl",
51
+ "manifestPath": "./.ekairos/runtime.json"
52
+ }
53
+ ```
54
+
55
+ ### `complete_dataset.mjs`
56
+
57
+ ```json
58
+ {
59
+ "datasetId": "target_dataset_v1",
60
+ "outputPath": "./work/output.jsonl",
61
+ "title": "Target Dataset",
62
+ "organizationId": "org_123",
63
+ "sandboxId": "sandbox_123",
64
+ "sources": [{ "kind": "query" }],
65
+ "sourceKinds": ["query"],
66
+ "instructions": "Summarize the query",
67
+ "schema": { "title": "Row", "description": "One row", "schema": { "type": "object" } },
68
+ "analysis": { "mode": "summary" }
69
+ }
70
+ ```
71
+
72
+ ## Notes
73
+
74
+ - Local workspace files can be read directly; no special fetch script is required for them.
75
+ - Query normalization follows the dataset builder convention:
76
+ - a single top-level array becomes rows,
77
+ - a single top-level object becomes one row,
78
+ - multi-entity results become rows with `__entity`.
79
+ - `complete_dataset.mjs` expects the final output file to be JSONL with rows shaped like:
80
+
81
+ ```json
82
+ {"type":"row","data":{...}}
83
+ ```
@@ -0,0 +1,156 @@
1
+ import { mkdir, readFile, writeFile } from "node:fs/promises"
2
+ import { dirname } from "node:path"
3
+ import { randomUUID } from "node:crypto"
4
+
5
+ const DEFAULT_MANIFEST_PATH = process.env.EKAIROS_RUNTIME_MANIFEST_PATH || ".ekairos/runtime.json"
6
+ const INSTANT_API_BASE_URL = "https://api.instantdb.com"
7
+
8
+ export async function readJsonFile(filePath) {
9
+ const content = await readFile(filePath, "utf8")
10
+ return JSON.parse(content)
11
+ }
12
+
13
+ export async function writeJsonFile(filePath, value) {
14
+ await mkdir(dirname(filePath), { recursive: true })
15
+ await writeFile(filePath, JSON.stringify(value, null, 2), "utf8")
16
+ }
17
+
18
+ export async function readRuntimeManifest(manifestPath) {
19
+ return await readJsonFile(manifestPath || DEFAULT_MANIFEST_PATH)
20
+ }
21
+
22
+ function authorizedHeaders(manifest) {
23
+ const headers = {
24
+ "content-type": "application/json",
25
+ "app-id": String(manifest?.instant?.appId ?? ""),
26
+ }
27
+ const scopedToken = String(manifest?.instant?.token ?? "").trim()
28
+ if (scopedToken) {
29
+ headers["as-token"] = scopedToken
30
+ }
31
+ return headers
32
+ }
33
+
34
+ async function jsonOrThrow(response) {
35
+ const text = await response.text()
36
+ let parsed = {}
37
+ try {
38
+ parsed = text ? JSON.parse(text) : {}
39
+ } catch {
40
+ parsed = { raw: text }
41
+ }
42
+ if (!response.ok) {
43
+ throw new Error(JSON.stringify({ status: response.status, body: parsed }))
44
+ }
45
+ return parsed
46
+ }
47
+
48
+ export async function instantQuery(query, manifestPath) {
49
+ const manifest = await readRuntimeManifest(manifestPath)
50
+ const response = await fetch(`${manifest?.instant?.apiBaseUrl || INSTANT_API_BASE_URL}/admin/query`, {
51
+ method: "POST",
52
+ headers: authorizedHeaders(manifest),
53
+ body: JSON.stringify({ query }),
54
+ })
55
+ return await jsonOrThrow(response)
56
+ }
57
+
58
+ export async function instantTransact(steps, manifestPath) {
59
+ const manifest = await readRuntimeManifest(manifestPath)
60
+ const response = await fetch(`${manifest?.instant?.apiBaseUrl || INSTANT_API_BASE_URL}/admin/transact`, {
61
+ method: "POST",
62
+ headers: authorizedHeaders(manifest),
63
+ body: JSON.stringify({
64
+ steps,
65
+ "throw-on-missing-attrs?": true,
66
+ }),
67
+ })
68
+ return await jsonOrThrow(response)
69
+ }
70
+
71
+ export async function instantUploadFile(params) {
72
+ const manifest = await readRuntimeManifest(params.manifestPath)
73
+ const headers = {
74
+ "app-id": String(manifest?.instant?.appId ?? ""),
75
+ path: String(params.path),
76
+ }
77
+ const scopedToken = String(manifest?.instant?.token ?? "").trim()
78
+ if (scopedToken) {
79
+ headers["as-token"] = scopedToken
80
+ }
81
+ if (params.contentType) {
82
+ headers["content-type"] = params.contentType
83
+ }
84
+ if (params.contentDisposition) {
85
+ headers["content-disposition"] = params.contentDisposition
86
+ }
87
+ const response = await fetch(`${manifest?.instant?.apiBaseUrl || INSTANT_API_BASE_URL}/admin/storage/upload`, {
88
+ method: "PUT",
89
+ headers,
90
+ body: params.buffer,
91
+ })
92
+ return await jsonOrThrow(response)
93
+ }
94
+
95
+ export function normalizeQueryRows(result) {
96
+ if (!result || typeof result !== "object") return []
97
+ const entries = Object.entries(result)
98
+ if (entries.length === 0) return []
99
+
100
+ if (entries.length === 1) {
101
+ const [key, value] = entries[0]
102
+ if (Array.isArray(value)) {
103
+ return value.map((row) => (row && typeof row === "object" ? row : { value: row }))
104
+ }
105
+ if (value && typeof value === "object") {
106
+ return [value]
107
+ }
108
+ return [{ [key]: value }]
109
+ }
110
+
111
+ const rows = []
112
+ for (const [key, value] of entries) {
113
+ if (Array.isArray(value)) {
114
+ for (const row of value) {
115
+ if (row && typeof row === "object") {
116
+ rows.push({ __entity: key, ...row })
117
+ } else {
118
+ rows.push({ __entity: key, value: row })
119
+ }
120
+ }
121
+ continue
122
+ }
123
+ if (value && typeof value === "object") {
124
+ rows.push({ __entity: key, ...value })
125
+ continue
126
+ }
127
+ rows.push({ __entity: key, value })
128
+ }
129
+ return rows
130
+ }
131
+
132
+ export function rowsToJsonl(rows) {
133
+ return rows
134
+ .map((row) => JSON.stringify({ type: "row", data: row }))
135
+ .join("\n")
136
+ .concat(rows.length > 0 ? "\n" : "")
137
+ }
138
+
139
+ export function countJsonlRows(text) {
140
+ return String(text ?? "")
141
+ .split("\n")
142
+ .map((line) => line.trim())
143
+ .filter(Boolean)
144
+ .map((line) => {
145
+ try {
146
+ return JSON.parse(line)
147
+ } catch {
148
+ return null
149
+ }
150
+ })
151
+ .filter((entry) => entry?.type === "row").length
152
+ }
153
+
154
+ export function newId() {
155
+ return randomUUID()
156
+ }
@@ -0,0 +1,104 @@
1
+ import { readFile } from "node:fs/promises"
2
+ import path from "node:path"
3
+ import {
4
+ countJsonlRows,
5
+ instantQuery,
6
+ instantTransact,
7
+ instantUploadFile,
8
+ newId,
9
+ readJsonFile,
10
+ } from "./_runtime.mjs"
11
+
12
+ const inputPath = process.argv[2]
13
+ if (!inputPath) {
14
+ console.error("complete_dataset_input_path_required")
15
+ process.exit(1)
16
+ }
17
+
18
+ const input = await readJsonFile(inputPath)
19
+ const datasetId = String(input.datasetId ?? "").trim()
20
+ const outputPath = String(input.outputPath ?? "").trim()
21
+
22
+ if (!datasetId) {
23
+ console.error("complete_dataset_dataset_id_required")
24
+ process.exit(1)
25
+ }
26
+ if (!outputPath) {
27
+ console.error("complete_dataset_output_path_required")
28
+ process.exit(1)
29
+ }
30
+
31
+ const fileBuffer = await readFile(outputPath)
32
+ const fileName = path.basename(outputPath) || "output.jsonl"
33
+ const storagePath = `/dataset/${datasetId}/${fileName}`
34
+ const uploaded = await instantUploadFile({
35
+ path: storagePath,
36
+ buffer: fileBuffer,
37
+ contentType: "application/x-ndjson",
38
+ contentDisposition: fileName,
39
+ manifestPath: input.manifestPath,
40
+ })
41
+
42
+ const uploadedFileId = String(uploaded?.data?.id ?? uploaded?.id ?? "").trim()
43
+ if (!uploadedFileId) {
44
+ console.error("complete_dataset_uploaded_file_id_missing")
45
+ process.exit(1)
46
+ }
47
+
48
+ const existing = await instantQuery(
49
+ {
50
+ dataset_datasets: {
51
+ $: { where: { datasetId }, limit: 1 },
52
+ },
53
+ },
54
+ input.manifestPath,
55
+ )
56
+
57
+ const current = Array.isArray(existing?.dataset_datasets) ? existing.dataset_datasets[0] : null
58
+ const entityId = String(current?.id ?? newId())
59
+ const rowCount = countJsonlRows(fileBuffer.toString("utf8"))
60
+ const createdAt = Number(current?.createdAt ?? Date.now())
61
+
62
+ const steps = [
63
+ [
64
+ "update",
65
+ "dataset_datasets",
66
+ entityId,
67
+ {
68
+ datasetId,
69
+ sandboxId: input.sandboxId ?? current?.sandboxId ?? null,
70
+ title: input.title ?? current?.title ?? datasetId,
71
+ status: "completed",
72
+ organizationId: input.organizationId ?? current?.organizationId ?? null,
73
+ instructions: input.instructions ?? current?.instructions ?? "",
74
+ sources: input.sources ?? current?.sources ?? [],
75
+ sourceKinds: input.sourceKinds ?? current?.sourceKinds ?? [],
76
+ analysis: input.analysis ?? current?.analysis ?? null,
77
+ schema: input.schema ?? current?.schema ?? null,
78
+ calculatedTotalRows: rowCount,
79
+ actualGeneratedRowCount: rowCount,
80
+ createdAt,
81
+ updatedAt: Date.now(),
82
+ },
83
+ ],
84
+ [
85
+ "link",
86
+ "dataset_datasets",
87
+ entityId,
88
+ {
89
+ dataFile: uploadedFileId,
90
+ },
91
+ ],
92
+ ]
93
+
94
+ const result = await instantTransact(steps, input.manifestPath)
95
+ process.stdout.write(
96
+ JSON.stringify({
97
+ datasetId,
98
+ entityId,
99
+ uploadedFileId,
100
+ rowCount,
101
+ storagePath,
102
+ result,
103
+ }),
104
+ )
@@ -0,0 +1,51 @@
1
+ import { mkdir, writeFile } from "node:fs/promises"
2
+ import { dirname } from "node:path"
3
+ import { instantQuery, readJsonFile } from "./_runtime.mjs"
4
+
5
+ const inputPath = process.argv[2]
6
+ if (!inputPath) {
7
+ console.error("dataset_source_input_path_required")
8
+ process.exit(1)
9
+ }
10
+
11
+ const input = await readJsonFile(inputPath)
12
+ const datasetId = String(input.datasetId ?? "").trim()
13
+ const outputPath = String(input.outputPath ?? "").trim()
14
+
15
+ if (!datasetId) {
16
+ console.error("dataset_source_dataset_id_required")
17
+ process.exit(1)
18
+ }
19
+ if (!outputPath) {
20
+ console.error("dataset_source_output_path_required")
21
+ process.exit(1)
22
+ }
23
+
24
+ const result = await instantQuery(
25
+ {
26
+ dataset_datasets: {
27
+ $: { where: { datasetId }, limit: 1 },
28
+ dataFile: {},
29
+ },
30
+ },
31
+ input.manifestPath,
32
+ )
33
+
34
+ const row = Array.isArray(result?.dataset_datasets) ? result.dataset_datasets[0] : null
35
+ const linkedFile = Array.isArray(row?.dataFile) ? row.dataFile[0] : row?.dataFile
36
+ const url = String(linkedFile?.url ?? "").trim()
37
+ if (!url) {
38
+ console.error("dataset_source_file_url_missing")
39
+ process.exit(1)
40
+ }
41
+
42
+ const response = await fetch(url)
43
+ if (!response.ok) {
44
+ console.error(`dataset_source_download_failed:${response.status}`)
45
+ process.exit(1)
46
+ }
47
+
48
+ const text = await response.text()
49
+ await mkdir(dirname(outputPath), { recursive: true })
50
+ await writeFile(outputPath, text, "utf8")
51
+ process.stdout.write(JSON.stringify({ outputPath, bytes: text.length }))
@@ -0,0 +1,22 @@
1
+ import { mkdir, writeFile } from "node:fs/promises"
2
+ import { dirname } from "node:path"
3
+ import { instantQuery, normalizeQueryRows, rowsToJsonl, readJsonFile } from "./_runtime.mjs"
4
+
5
+ const inputPath = process.argv[2]
6
+ if (!inputPath) {
7
+ console.error("query_input_path_required")
8
+ process.exit(1)
9
+ }
10
+
11
+ const input = await readJsonFile(inputPath)
12
+ const outputPath = String(input.outputPath ?? "").trim()
13
+ if (!outputPath) {
14
+ console.error("query_output_path_required")
15
+ process.exit(1)
16
+ }
17
+
18
+ const result = await instantQuery(input.query ?? {}, input.manifestPath)
19
+ const rows = normalizeQueryRows(result)
20
+ await mkdir(dirname(outputPath), { recursive: true })
21
+ await writeFile(outputPath, rowsToJsonl(rows), "utf8")
22
+ process.stdout.write(JSON.stringify({ outputPath, rowCount: rows.length }))
@@ -0,0 +1,13 @@
1
+ name = "dataset"
2
+ version = "0.1.0"
3
+ description = "Materialize and persist datasets using the Ekairos runtime manifest."
4
+ kind = "action"
5
+ namespace = "ekairos"
6
+ tags = ["dataset", "ekairos", "instantdb", "runtime"]
7
+
8
+ [runtime]
9
+ language = "javascript"
10
+ entrypoint = "code/complete_dataset.mjs"
11
+
12
+ [permissions]
13
+ network = ["https://api.instantdb.com", "https://storage.instantdb.com", "https://*"]