@pipedream/openai 0.5.0 → 0.5.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pipedream/openai",
-  "version": "0.5.0",
+  "version": "0.5.2",
   "description": "Pipedream OpenAI Components",
   "main": "openai.app.mjs",
   "keywords": [

package/sources/{common.mjs → common/common.mjs}

@@ -1,5 +1,5 @@
 import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform";
-import openai from "../openai.app.mjs";
+import openai from "../../openai.app.mjs";
 
 export default {
   props: {
@@ -13,11 +13,11 @@ export default {
     },
   },
   methods: {
-    _getSavedItems() {
-      return this.db.get("savedItems") ?? [];
+    _getLastCreated() {
+      return this.db.get("lastCreated") || 0;
     },
-    _setSavedItems(value) {
-      this.db.set("savedItems", value);
+    _setLastCreated(lastCreated) {
+      this.db.set("lastCreated", lastCreated);
     },
     getMeta() {
       throw new Error("No item metadata implemented!");
@@ -26,18 +26,20 @@ export default {
       throw new Error("No item fetching implemented!");
     },
     async getAndProcessItems(maxEvents) {
-      const savedItems = this._getSavedItems();
+      const lastCreated = this._getLastCreated();
       const { data } = await this.getData();
+      if (!data?.length) {
+        return;
+      }
+      this._setLastCreated(data[0].created_at);
       data
-        ?.filter(({ id }) => !savedItems.includes(id))
+        ?.filter(({ created_at }) => created_at >= lastCreated)
         .reverse()
         .forEach((item, index) => {
           if (!maxEvents || index < maxEvents) {
             this.$emit(item, this.getMeta(item));
           }
-          savedItems.push(item.id);
         });
-      this._setSavedItems(savedItems);
     },
   },
   hooks: {

package/sources/new-batch-completed/new-batch-completed.mjs

@@ -0,0 +1,46 @@
+import common from "../common/common.mjs";
+import sampleEmit from "./test-event.mjs";
+
+export default {
+  ...common,
+  key: "openai-new-batch-completed",
+  name: "New Batch Completed",
+  description: "Emit new event when a new batch is completed in OpenAI. [See the documentation](https://platform.openai.com/docs/api-reference/batch/list)",
+  version: "0.0.1",
+  type: "source",
+  dedupe: "unique",
+  methods: {
+    ...common.methods,
+    _getLastCompleted() {
+      return this.db.get("lastCompleted") || 0;
+    },
+    _setLastCompleted(lastCompleted) {
+      this.db.set("lastCompleted", lastCompleted);
+    },
+    getMeta(batch) {
+      return {
+        id: batch.id,
+        summary: `Batch Completed with ID ${batch.id}`,
+        ts: batch.completed_at * 1000,
+      };
+    },
+    async getAndProcessItems(max) {
+      const lastCompleted = this._getLastCompleted();
+      let maxCompleted = lastCompleted;
+      const results = this.openai.paginate({
+        resourceFn: this.openai.listBatches,
+        max,
+      });
+      const batches = [];
+      for await (const batch of results) {
+        if (batch.completed_at && batch.completed_at > lastCompleted) {
+          batches.push(batch);
+          maxCompleted = Math.max(batch.completed_at, maxCompleted);
+        }
+      }
+      this._setLastCompleted(maxCompleted);
+      batches.reverse().forEach((item) => this.$emit(item, this.getMeta(item)));
+    },
+  },
+  sampleEmit,
+};

package/sources/new-batch-completed/test-event.mjs

@@ -0,0 +1,29 @@
+export default {
+  "id": "batch_abc123",
+  "object": "batch",
+  "endpoint": "/v1/completions",
+  "errors": null,
+  "input_file_id": "file-abc123",
+  "completion_window": "24h",
+  "status": "completed",
+  "output_file_id": "file-cvaTdG",
+  "error_file_id": "file-HOWS94",
+  "created_at": 1711471533,
+  "in_progress_at": 1711471538,
+  "expires_at": 1711557933,
+  "finalizing_at": 1711493133,
+  "completed_at": 1711493163,
+  "failed_at": null,
+  "expired_at": null,
+  "cancelling_at": null,
+  "cancelled_at": null,
+  "request_counts": {
+    "total": 100,
+    "completed": 95,
+    "failed": 5
+  },
+  "metadata": {
+    "customer_id": "user_123456789",
+    "batch_description": "Nightly eval job",
+  }
+}

package/sources/new-file-created/new-file-created.mjs

@@ -1,11 +1,12 @@
-import common from "../common.mjs";
+import common from "../common/common.mjs";
+import sampleEmit from "./test-event.mjs";
 
 export default {
   ...common,
   key: "openai-new-file-created",
   name: "New File Created",
   description: "Emit new event when a new file is created in OpenAI. [See the documentation](https://platform.openai.com/docs/api-reference/files/list)",
-  version: "0.0.5",
+  version: "0.0.6",
   type: "source",
   dedupe: "unique",
   props: {
@@ -21,7 +22,7 @@ export default {
   },
   methods: {
     ...common.methods,
-    async getData() {
+    getData() {
       return this.openai.listFiles({
         purpose: this.purpose,
       });
@@ -34,4 +35,5 @@ export default {
       };
     },
   },
+  sampleEmit,
 };

package/sources/new-file-created/test-event.mjs

@@ -0,0 +1,10 @@
+export default {
+  "id": "file-abc123",
+  "object": "file",
+  "bytes": 140,
+  "created_at": 1613779121,
+  "filename": "puppy.jsonl",
+  "purpose": "fine-tune",
+  "status": "processed",
+  "status_details": null
+}

package/sources/new-fine-tuning-job-created/new-fine-tuning-job-created.mjs

@@ -1,16 +1,17 @@
-import common from "../common.mjs";
+import common from "../common/common.mjs";
+import sampleEmit from "./test-event.mjs";
 
 export default {
   ...common,
   key: "openai-new-fine-tuning-job-created",
   name: "New Fine Tuning Job Created",
   description: "Emit new event when a new fine-tuning job is created in OpenAI. [See the documentation](https://platform.openai.com/docs/api-reference/fine-tuning/list)",
-  version: "0.0.5",
+  version: "0.0.6",
   type: "source",
   dedupe: "unique",
   methods: {
     ...common.methods,
-    async getData() {
+    getData() {
       return this.openai.listFineTuningJobs();
     },
     getMeta(item) {
@@ -21,4 +22,5 @@ export default {
       };
     },
   },
+  sampleEmit,
 };

package/sources/new-fine-tuning-job-created/test-event.mjs

@@ -0,0 +1,19 @@
+export default {
+  "object": "fine_tuning.job",
+  "id": "ftjob-vbCfZSaivshuV18SjpCQwu2I",
+  "model": "gpt-3.5-turbo-0125",
+  "created_at": 1715284148,
+  "finished_at": null,
+  "fine_tuned_model": null,
+  "organization_id": "org-IQsNYSuAzzwvzXiKFGa1BeR4",
+  "result_files": [],
+  "status": "failed",
+  "validation_file": null,
+  "training_file": "file-D1svkZhYUtI4E8IpYsmIs3qY",
+  "hyperparameters": null,
+  "trained_tokens": null,
+  "user_provided_suffix": null,
+  "seed": 66573047,
+  "estimated_finish": null,
+  "integrations": []
+}

package/sources/new-run-state-changed/new-run-state-changed.mjs

@@ -1,11 +1,12 @@
-import common from "../common.mjs";
+import common from "../common/common.mjs";
+import sampleEmit from "./test-event.mjs";
 
 export default {
   ...common,
   key: "openai-new-run-state-changed",
   name: "New Run State Changed",
   description: "Emit new event every time a run changes its status. [See the documentation](https://platform.openai.com/docs/api-reference/runs/listRuns)",
-  version: "0.0.1",
+  version: "0.0.2",
   type: "source",
   dedupe: "unique",
   props: {
@@ -81,4 +82,5 @@ export default {
       setStatusItems(buildStatusItems(data));
     },
   },
+  sampleEmit,
 };

package/sources/new-run-state-changed/test-event.mjs

@@ -0,0 +1,36 @@
+export default {
+  "id": "run_RW7H8ICHslGIXBCBpgXn4OJy",
+  "object": "thread.run",
+  "created_at": 1715284963,
+  "assistant_id": "asst_aHkRGOagtHiTJJF7hNqLqcrv",
+  "thread_id": "thread_IqZAvwcU3SVsLeFeJ7fkx6LN",
+  "status": "completed",
+  "started_at": 1715284963,
+  "expires_at": null,
+  "cancelled_at": null,
+  "failed_at": null,
+  "completed_at": 1715284966,
+  "required_action": null,
+  "last_error": null,
+  "model": "gpt-4-turbo",
+  "instructions": "",
+  "tools": [],
+  "file_ids": [],
+  "metadata": {},
+  "temperature": 1,
+  "top_p": 1,
+  "max_completion_tokens": null,
+  "max_prompt_tokens": null,
+  "truncation_strategy": {
+    "type": "auto",
+    "last_messages": null
+  },
+  "incomplete_details": null,
+  "usage": {
+    "prompt_tokens": 25,
+    "completion_tokens": 25,
+    "total_tokens": 50
+  },
+  "response_format": "auto",
+  "tool_choice": "auto"
+}

package/actions/common/constants.mjs

@@ -1,14 +0,0 @@
-export default {
-  IMAGE_SIZES: [
-    "256x256",
-    "512x512",
-    "1024x1024",
-  ],
-  TRANSCRIPTION_FORMATS: [
-    "json",
-    "text",
-    "srt",
-    "verbose_json",
-    "vtt",
-  ],
-};

package/actions/create-message/create-message.mjs

@@ -1,64 +0,0 @@
-import openai from "../../openai.app.mjs";
-
-export default {
-  key: "openai-create-message",
-  name: "Create Message (Assistants)",
-  description: "Create a message in a thread. [See the documentation](https://platform.openai.com/docs/api-reference/messages/createMessage)",
-  version: "0.0.7",
-  type: "action",
-  props: {
-    openai,
-    threadId: {
-      propDefinition: [
-        openai,
-        "threadId",
-      ],
-    },
-    content: {
-      propDefinition: [
-        openai,
-        "content",
-      ],
-    },
-    role: {
-      propDefinition: [
-        openai,
-        "role",
-      ],
-      default: "user",
-    },
-    fileIds: {
-      propDefinition: [
-        openai,
-        "fileIds",
-      ],
-      optional: true,
-    },
-    metadata: {
-      propDefinition: [
-        openai,
-        "metadata",
-      ],
-      optional: true,
-    },
-  },
-  async run({ $ }) {
-    const fileIdsArray = this.fileIds
-      ? this.fileIds.map((fileId) => fileId.trim())
-      : undefined;
-    const metadataObject = this.metadata
-      ? JSON.parse(this.metadata)
-      : undefined;
-
-    const response = await this.openai.createMessage({
-      threadId: this.threadId,
-      content: this.content,
-      role: this.role,
-      fileIds: fileIdsArray,
-      metadata: metadataObject,
-    });
-
-    $.export("$summary", `Successfully created a message in thread ${this.threadId}`);
-    return response;
-  },
-};

package/actions/create-run/create-run.mjs

@@ -1,65 +0,0 @@
-import { parseToolsArray } from "../../common/helpers.mjs";
-import openai from "../../openai.app.mjs";
-
-export default {
-  key: "openai-create-run",
-  name: "Create Run (Assistants)",
-  description: "Creates a run given a thread ID and assistant ID. [See the documentation](https://platform.openai.com/docs/api-reference/runs/create)",
-  version: "0.1.3",
-  type: "action",
-  props: {
-    openai,
-    threadId: {
-      propDefinition: [
-        openai,
-        "threadId",
-      ],
-    },
-    assistantId: {
-      propDefinition: [
-        openai,
-        "assistantId",
-      ],
-    },
-    model: {
-      propDefinition: [
-        openai,
-        "model",
-      ],
-    },
-    instructions: {
-      propDefinition: [
-        openai,
-        "instructions",
-      ],
-    },
-    tools: {
-      propDefinition: [
-        openai,
-        "tools",
-      ],
-    },
-    metadata: {
-      propDefinition: [
-        openai,
-        "metadata",
-      ],
-    },
-  },
-  async run({ $ }) {
-    const response = await this.openai.createRun({
-      threadId: this.threadId,
-      assistantId: this.assistantId,
-      model: this.model,
-      instructions: this.instructions,
-      tools: parseToolsArray(this.tools),
-      metadata: this.metadata,
-    });
-
-    const summary = response.id
-      ? `Successfully created a run with ID: ${response.id}`
-      : `Successfully created a run in thread ${this.threadId}`;
-    $.export("$summary", summary);
-    return response;
-  },
-};

package/actions/create-thread-and-run/create-thread-and-run.mjs

@@ -1,62 +0,0 @@
-import { parseToolsArray } from "../../common/helpers.mjs";
-import openai from "../../openai.app.mjs";
-
-export default {
-  key: "openai-create-thread-and-run",
-  name: "Create Thread and Run (Assistants)",
-  description: "Create a thread and run it in one request using the specified assistant ID and optional parameters. [See the documentation](https://platform.openai.com/docs/api-reference)",
-  version: "0.1.3",
-  type: "action",
-  props: {
-    openai,
-    assistantId: {
-      propDefinition: [
-        openai,
-        "assistantId",
-      ],
-    },
-    thread: {
-      type: "object",
-      label: "Thread",
-      description: "The thread object containing messages and other optional properties.",
-      optional: true,
-    },
-    model: {
-      propDefinition: [
-        openai,
-        "model",
-      ],
-    },
-    instructions: {
-      propDefinition: [
-        openai,
-        "instructions",
-      ],
-    },
-    tools: {
-      propDefinition: [
-        openai,
-        "tools",
-      ],
-    },
-    metadata: {
-      propDefinition: [
-        openai,
-        "metadata",
-      ],
-    },
-  },
-  async run({ $ }) {
-    const response = await this.openai.createThreadAndRun({
-      assistant_id: this.assistantId,
-      thread: this.thread,
-      model: this.model,
-      instructions: this.instructions,
-      tools: parseToolsArray(this.tools),
-      metadata: this.metadata,
-    });
-
-    $.export("$summary", `Successfully created thread and initiated run with ID: ${response.id}`);
-    return response;
-  },
-};

package/actions/modify-message/modify-message.mjs

@@ -1,42 +0,0 @@
-import openai from "../../openai.app.mjs";
-
-export default {
-  key: "openai-modify-message",
-  name: "Modify Message (Assistants)",
-  description: "Modifies an existing message in a thread. [See the documentation](https://platform.openai.com/docs/api-reference/messages/modifyMessage)",
-  version: "0.0.7",
-  type: "action",
-  props: {
-    openai,
-    threadId: {
-      propDefinition: [
-        openai,
-        "threadId",
-      ],
-    },
-    messageId: {
-      propDefinition: [
-        openai,
-        "messageId",
-      ],
-    },
-    metadata: {
-      type: "string",
-      label: "Metadata",
-      description: "Metadata for the message in JSON string format",
-      optional: true,
-    },
-  },
-  async run({ $ }) {
-    const response = await this.openai.modifyMessage({
-      threadId: this.threadId,
-      messageId: this.messageId,
-      metadata: this.metadata
-        ? JSON.parse(this.metadata)
-        : undefined,
-    });
-
-    $.export("$summary", `Successfully modified message ${this.messageId} in thread ${this.threadId}`);
-    return response;
-  },
-};

package/actions/modify-run/modify-run.mjs

@@ -1,45 +0,0 @@
-import openai from "../../openai.app.mjs";
-
-export default {
-  key: "openai-modify-run",
-  name: "Modify Run (Assistants)",
-  description: "Modifies an existing run. [See the documentation](https://platform.openai.com/docs/api-reference/runs/modifyRun)",
-  version: "0.0.6",
-  type: "action",
-  props: {
-    openai,
-    threadId: {
-      propDefinition: [
-        openai,
-        "threadId",
-      ],
-    },
-    runId: {
-      propDefinition: [
-        openai,
-        "runId",
-        ({ threadId }) => ({
-          threadId,
-        }),
-      ],
-    },
-    metadata: {
-      propDefinition: [
-        openai,
-        "metadata",
-      ],
-    },
-  },
-  async run({ $ }) {
-    const response = await this.openai.modifyRun({
-      threadId: this.threadId,
-      runId: this.runId,
-      ...(this.metadata && {
-        metadata: this.metadata,
-      }),
-    });
-
-    $.export("$summary", `Successfully modified run with ID: ${this.runId}`);
-    return response;
-  },
-};