@pipedream/openai 0.1.10 → 0.3.0

package/package.json

@@ -1,8 +1,8 @@
 {
   "name": "@pipedream/openai",
-  "version": "0.1.10",
+  "version": "0.3.0",
   "description": "Pipedream OpenAI Components",
-  "main": "app/openai.app.mjs",
+  "main": "openai.app.mjs",
   "keywords": [
     "pipedream",
     "openai"
@@ -18,7 +18,8 @@
     "@pipedream/platform": "^1.2.1",
     "@pipedream/types": "^0.1.4",
     "got": "^12.6.0",
-    "openai": "^3.2.1"
+    "openai": "^3.2.1",
+    "form-data": "^4.0.0"
   },
   "devDependencies": {
     "@types/node": "^17.0.45"

package/sources/common.mjs

@@ -0,0 +1,51 @@
+import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform";
+import openai from "../openai.app.mjs";
+
+export default {
+  props: {
+    openai,
+    db: "$.service.db",
+    timer: {
+      type: "$.interface.timer",
+      default: {
+        intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL,
+      },
+    },
+  },
+  methods: {
+    _getSavedItems() {
+      return this.db.get("savedItems") ?? [];
+    },
+    _setSavedItems(value) {
+      this.db.set("savedItems", value);
+    },
+    getMeta() {
+      throw new Error("No item metadata implemented!");
+    },
+    async getData() {
+      throw new Error("No item fetching implemented!");
+    },
+    async getAndProcessItems(maxEvents) {
+      const savedItems = this._getSavedItems();
+      const { data } = await this.getData();
+      data
+        ?.filter(({ id }) => !savedItems.includes(id))
+        .reverse()
+        .forEach((item, index) => {
+          if (!maxEvents || index < maxEvents) {
+            this.$emit(item, this.getMeta(item));
+          }
+          savedItems.push(item.id);
+        });
+      this._setSavedItems(savedItems);
+    },
+  },
+  hooks: {
+    async deploy() {
+      await this.getAndProcessItems(10);
+    },
+  },
+  async run() {
+    await this.getAndProcessItems();
+  },
+};

package/sources/new-file-created/new-file-created.mjs

@@ -0,0 +1,38 @@
+import openai from "../../openai.app.mjs";
+import common from "../common.mjs";
+
+export default {
+  ...common,
+  key: "openai-new-file-created",
+  name: "New File Created",
+  description: "Emit new event when a new file is created in OpenAI. [See the documentation](https://platform.openai.com/docs/api-reference/files/list)",
+  version: "0.0.1",
+  type: "source",
+  dedupe: "unique",
+  props: {
+    ...common.props,
+    purpose: {
+      propDefinition: [
+        openai,
+        "purpose",
+      ],
+      description: "If specified, events will only be emitted for files with the specified purpose.",
+      optional: true,
+    },
+  },
+  methods: {
+    ...common.methods,
+    async getData() {
+      return this.openai.listFiles({
+        purpose: this.purpose,
+      });
+    },
+    getMeta(item) {
+      return {
+        id: item.id,
+        summary: `New File: ${item.filename}`,
+        ts: item.created_at * 1000,
+      };
+    },
+  },
+};

package/sources/new-fine-tuning-job-created/new-fine-tuning-job-created.mjs

@@ -0,0 +1,24 @@
+import common from "../common.mjs";
+
+export default {
+  ...common,
+  key: "openai-new-fine-tuning-job-created",
+  name: "New Fine Tuning Job Created",
+  description: "Emit new event when a new fine-tuning job is created in OpenAI. [See the documentation](https://platform.openai.com/docs/api-reference/fine-tuning/list)",
+  version: "0.0.1",
+  type: "source",
+  dedupe: "unique",
+  methods: {
+    ...common.methods,
+    async getData() {
+      return this.openai.listFineTuningJobs();
+    },
+    getMeta(item) {
+      return {
+        id: item.id,
+        summary: `New Fine Tuning Job: ${item.id}`,
+        ts: item.created_at * 1000,
+      };
+    },
+  },
+};

package/app/openai.app.mjs

@@ -1,180 +0,0 @@
-import { axios } from "@pipedream/platform";
-
-export default {
-  type: "app",
-  app: "openai",
-  propDefinitions: {
-    completionModelId: {
-      label: "Model",
-      description: "The ID of the model to use for completions. **This action doesn't support the ChatGPT `turbo` models**. Use the **Chat** action for those, instead.",
-      type: "string",
-      async options() {
-        return (await this.getCompletionModels({})).map((model) => model.id);
-      },
-      default: "text-davinci-003",
-    },
-    chatCompletionModelId: {
-      label: "Model",
-      description: "The ID of the model to use for chat completions",
-      type: "string",
-      async options() {
-        return (await this.getChatCompletionModels({})).map((model) => model.id);
-      },
-      default: "gpt-3.5-turbo",
-    },
-    embeddingsModelId: {
-      label: "Model",
-      description: "The ID of the embeddings model to use. OpenAI recommends using `text-embedding-ada-002` for nearly all use cases: \"It's better, cheaper, and simpler to use. [Read the blog post announcement](https://openai.com/blog/new-and-improved-embedding-model)\".",
-      type: "string",
-      async options() {
-        return (await this.getEmbeddingsModels({})).map((model) => model.id);
-      },
-      default: "text-embedding-ada-002",
-    },
-  },
-  methods: {
-    _apiKey() {
-      return this.$auth.api_key;
-    },
-    _baseApiUrl() {
-      return "https://api.openai.com/v1";
-    },
-    _commonHeaders() {
-      return {
-        "Authorization": `Bearer ${this._apiKey()}`,
-        "Accept": "application/json",
-        "User-Agent": "@PipedreamHQ/pipedream v1.0",
-      };
-    },
-    async _makeRequest({
-      $ = this,
-      path,
-      ...args
-    } = {}) {
-      return axios($, {
-        url: `${this._baseApiUrl()}${path}`,
-        headers: {
-          ...this._commonHeaders(),
-        },
-        maxBodyLength: Infinity,
-        ...args,
-      });
-    },
-    async models({ $ }) {
-      const { data: models } = await this._makeRequest({
-        $,
-        path: "/models",
-      });
-      return models.sort((a, b) => a?.id.localeCompare(b?.id));
-    },
-    async getChatCompletionModels({ $ }) {
-      const models = await this.models({
-        $,
-      });
-      return models.filter((model) => model.id.match(/turbo|gpt/gi));
-    },
-    async getCompletionModels({ $ }) {
-      const models = await this.models({
-        $,
-      });
-      return models.filter((model) => {
-        const { id } = model;
-        return (
-          id.match(/^(?=.*\b(babbage|davinci|ada|curie)\b)(?!.*\b(whisper|turbo|edit|insert|search|embedding|similarity|001)\b).*$/gm)
-        );
-      });
-    },
-    async getEmbeddingsModels({ $ }) {
-      const models = await this.models({
-        $,
-      });
-      return models.filter((model) => {
-        const { id } = model;
-        return (
-          id.match(/^(text-embedding-ada-002|.*-(davinci|curie|babbage|ada)-.*-001)$/gm)
-        );
-      });
-    },
-    async _makeCompletion({
-      $, path, args,
-    }) {
-      const data = await this._makeRequest({
-        $,
-        path,
-        method: "POST",
-        data: args,
-      });
-
-      // For completions, return the text of the first choice at the top-level
-      let generated_text;
-      if (path === "/completions") {
-        const { choices } = data;
-        generated_text = choices?.[0]?.text;
-      }
-      // For chat completions, return the assistant message at the top-level
-      let generated_message;
-      if (path === "/chat/completions") {
-        const { choices } = data;
-        generated_message = choices?.[0]?.message;
-      }
-
-      return {
-        generated_text,
-        generated_message,
-        ...data,
-      };
-    },
-    async createCompletion({
-      $, args,
-    }) {
-      return this._makeCompletion({
-        $,
-        path: "/completions",
-        args,
-      });
-    },
-    async createChatCompletion({
-      $, args,
-    }) {
-      return this._makeCompletion({
-        $,
-        path: "/chat/completions",
-        args,
-      });
-    },
-    async createImage({
-      $, args,
-    }) {
-      return this._makeRequest({
-        $,
-        path: "/images/generations",
-        data: args,
-        method: "POST",
-      });
-    },
-    async createEmbeddings({
-      $, args,
-    }) {
-      return this._makeRequest({
-        $,
-        path: "/embeddings",
-        data: args,
-        method: "POST",
-      });
-    },
-    async createTranscription({
-      $, form,
-    }) {
-      return this._makeRequest({
-        $,
-        path: "/audio/transcriptions",
-        method: "POST",
-        headers: {
-          ...this._commonHeaders(),
-          "Content-Type": `multipart/form-data; boundary=${form._boundary}`,
-        },
-        data: form,
-      });
-    },
-  },
-};