@upstart.gg/sdk 0.0.130 → 0.0.132

package/src/shared/datasources/external/mastodon/status/sample.array.ts

@@ -0,0 +1,59 @@
+import type { MastodonStatusArraySchema } from "./schema";
+
+export const sample = [
+  {
+    account: {
+      acct: "Gargron",
+      avatar: "https://files.mastodon.social/accounts/avatars/000/000/001/original/7b7e3c3e3e7e3e3e.png",
+      avatar_static:
+        "https://files.mastodon.social/accounts/avatars/000/000/001/original/7b7e3c3e3e7e3e3e.png",
+      bot: false,
+      created_at: "2016-04-04T17:41:54.000Z",
+      display_name: "Eugen",
+      emojis: [
+        {
+          category: "custom",
+          shortcode: "blobawave",
+          static_url: "https://files.mastodon.social/custom_emojis/images/000/000/001/static/1f4a9.png",
+          url: "https://files.mastodon.social/custom_emojis/images/000/000/001/original/1f4a9.png",
+          visible_in_picker: true,
+        },
+      ],
+      id: "1",
+      url: "https://mastodon.social/@Gargron",
+      username: "Gargron",
+      discoverable: true,
+      fields: [],
+      followers_count: 100,
+      following_count: 100,
+      group: false,
+      header: "https://files.mastodon.social/accounts/headers/000/000/001/original/7b7e3c3e3e7e3e3e.png",
+      header_static:
+        "https://files.mastodon.social/accounts/headers/000/000/001/original/7b7e3c3e3e7e3e3e.png",
+      last_status_at: "2021-06-26T18:00:00",
+      locked: false,
+      note: "Admin of mastodon.social",
+      noindex: false,
+      statuses_count: 100,
+      suspended: false,
+    },
+    content: "Hello, world!",
+    created_at: "2021-06-26T18:00:00",
+    id: "1",
+    media_attachments: [],
+    reblogs_count: 0,
+    replies_count: 0,
+    favourites_count: 0,
+    tags: [],
+    uri: "https://mastodon.social/@Gargron/1",
+    visibility: "public",
+    sensitive: false,
+    application: {
+      name: "Mastodon",
+      website: "https://joinmastodon.org",
+    },
+    mentions: [],
+    emojis: [],
+    url: "https://mastodon.social/@Gargron/1",
+  },
+] satisfies MastodonStatusArraySchema;

package/src/shared/datasources/external/mastodon/status/sample.single.ts

@@ -0,0 +1,55 @@
+import type { MastodonStatusSchema } from "./schema";
+
+export const sample = {
+  account: {
+    acct: "Gargron",
+    avatar: "https://files.mastodon.social/accounts/avatars/000/000/001/original/7b7e3c3e3e7e3e3e.png",
+    avatar_static: "https://files.mastodon.social/accounts/avatars/000/000/001/original/7b7e3c3e3e7e3e3e.png",
+    bot: false,
+    created_at: "2016-04-04T17:41:54.000Z",
+    display_name: "Eugen",
+    emojis: [
+      {
+        category: "custom",
+        shortcode: "blobawave",
+        static_url: "https://files.mastodon.social/custom_emojis/images/000/000/001/static/1f4a9.png",
+        url: "https://files.mastodon.social/custom_emojis/images/000/000/001/original/1f4a9.png",
+        visible_in_picker: true,
+      },
+    ],
+    id: "1",
+    url: "https://mastodon.social/@Gargron",
+    username: "Gargron",
+    discoverable: true,
+    fields: [],
+    followers_count: 100,
+    following_count: 100,
+    group: false,
+    header: "https://files.mastodon.social/accounts/headers/000/000/001/original/7b7e3c3e3e7e3e3e.png",
+    header_static: "https://files.mastodon.social/accounts/headers/000/000/001/original/7b7e3c3e3e7e3e3e.png",
+    last_status_at: "2021-06-26T18:00:00",
+    locked: false,
+    note: "Admin of mastodon.social",
+    noindex: false,
+    statuses_count: 100,
+    suspended: false,
+  },
+  content: "Hello, world!",
+  created_at: "2021-06-26T18:00:00",
+  id: "1",
+  media_attachments: [],
+  reblogs_count: 0,
+  replies_count: 0,
+  favourites_count: 0,
+  tags: [],
+  uri: "https://mastodon.social/@Gargron/1",
+  visibility: "public",
+  sensitive: false,
+  application: {
+    name: "Mastodon",
+    website: "https://joinmastodon.org",
+  },
+  mentions: [],
+  emojis: [],
+  url: "https://mastodon.social/@Gargron/1",
+} satisfies MastodonStatusSchema;

package/src/shared/datasources/external/mastodon/status/schema.ts

@@ -0,0 +1,125 @@
+import { Type, type Static } from "@sinclair/typebox";
+import { mastodonAccountSchema } from "../account/schema";
+
+const mastodonMediaAttachmentSchema = Type.Object({
+  id: Type.String(),
+  type: Type.Union([
+    Type.Literal("audio"),
+    Type.Literal("image"),
+    Type.Literal("video"),
+    Type.Literal("gifv"),
+    Type.Literal("unknown"),
+  ]),
+  url: Type.String(),
+  preview_url: Type.Optional(Type.String()),
+  remote_url: Type.Optional(Type.String()),
+  meta: Type.Record(Type.String(), Type.Any()),
+  description: Type.Optional(Type.String()),
+  blurhash: Type.Optional(Type.String()),
+});
+
+const mastodonEmojiSchema = Type.Object({
+  shortcode: Type.String(),
+  url: Type.String(),
+  static_url: Type.String(),
+  visible_in_picker: Type.Boolean(),
+  category: Type.Optional(Type.String()),
+});
+
+const mastodonPreviewCardSchema = Type.Object({
+  url: Type.String(),
+  title: Type.String(),
+  description: Type.String(),
+  type: Type.Union([
+    Type.Literal("link"),
+    Type.Literal("photo"),
+    Type.Literal("video"),
+    Type.Literal("rich"),
+  ]),
+  author_name: Type.String(),
+  author_url: Type.String(),
+  provider_name: Type.String(),
+  provider_url: Type.String(),
+  html: Type.String(),
+  width: Type.Number(),
+  height: Type.Number(),
+  image: Type.Optional(Type.String()),
+  embed_url: Type.String(),
+  blurhash: Type.Optional(Type.String()),
+});
+
+const mastodonPollSchema = Type.Object({
+  id: Type.String(),
+  expires_at: Type.String(),
+  expired: Type.Boolean(),
+  multiple: Type.Boolean(),
+  votes_count: Type.Number(),
+  options: Type.Array(
+    Type.Object({
+      title: Type.String(),
+      votes_count: Type.Number(),
+    }),
+  ),
+  voted: Type.Optional(Type.Boolean()),
+  own_votes: Type.Optional(Type.Array(Type.Number())),
+  emojis: Type.Array(mastodonEmojiSchema),
+});
+
+export const mastodonStatusSchema = Type.Object({
+  id: Type.String(),
+  uri: Type.String(),
+  created_at: Type.String(),
+  account: mastodonAccountSchema,
+  content: Type.String(),
+  visibility: Type.Union([
+    Type.Literal("public"),
+    Type.Literal("unlisted"),
+    Type.Literal("private"),
+    Type.Literal("direct"),
+  ]),
+  sensitive: Type.Boolean(),
+  media_attachments: Type.Array(mastodonMediaAttachmentSchema),
+  application: Type.Optional(
+    Type.Object({
+      name: Type.String(),
+      website: Type.Optional(Type.String()),
+    }),
+  ),
+  mentions: Type.Array(
+    Type.Object({
+      id: Type.String(),
+      username: Type.String(),
+      acct: Type.String(),
+      url: Type.String(),
+    }),
+  ),
+  tags: Type.Array(
+    Type.Object({
+      name: Type.String(),
+      url: Type.String(),
+    }),
+  ),
+  emojis: Type.Array(mastodonEmojiSchema),
+  reblogs_count: Type.Number(),
+  favourites_count: Type.Number(),
+  replies_count: Type.Number(),
+  url: Type.String(),
+  in_reply_to_id: Type.Optional(Type.String()),
+  in_reply_to_account_id: Type.Optional(Type.String()),
+  reblog: Type.Optional(Type.Ref("mastodonStatus")),
+  poll: Type.Optional(mastodonPollSchema),
+  card: Type.Optional(mastodonPreviewCardSchema),
+  language: Type.Optional(Type.String()),
+  text: Type.Optional(Type.String()),
+  edited_at: Type.Optional(Type.String()),
+  favourited: Type.Optional(Type.Boolean()),
+  reblogged: Type.Optional(Type.Boolean()),
+  muted: Type.Optional(Type.Boolean()),
+  bookmarked: Type.Optional(Type.Boolean()),
+  pinned: Type.Optional(Type.Boolean()),
+});
+
+export type MastodonStatusSchema = Static<typeof mastodonStatusSchema>;
+
+export const mastodonStatusArraySchema = Type.Array(mastodonStatusSchema);
+export type MastodonStatusArraySchema = Static<typeof mastodonStatusArraySchema>;

package/src/shared/datasources/external/mastodon/status/tests/fetcher.test.ts

@@ -0,0 +1,74 @@
+import { describe, it, expect, vi, beforeEach } from "vitest";
+import fetchMastodonStatus from "../fetcher";
+import { UnauthorizedError } from "~/shared/errors";
+import { sample } from "../sample.array";
+import fetchMastodonAccount from "../../account/fetcher";
+import { sample as accountSample } from "../../account/sample";
+
+// Mock the fetch function and fetchMastodonAccount
+global.fetch = vi.fn();
+// Mock the entire module
+vi.mock("../../account/fetcher", () => ({
+  default: vi.fn(),
+}));
+
+describe("fetchMastodonStatus", () => {
+  beforeEach(() => {
+    vi.resetAllMocks();
+  });
+
+  it("should fetch Mastodon statuses successfully", async () => {
+    (global.fetch as any).mockResolvedValueOnce({
+      ok: true,
+      json: () => Promise.resolve(sample),
+    });
+
+    vi.mocked(fetchMastodonAccount).mockResolvedValueOnce(accountSample);
+
+    const result = await fetchMastodonStatus({
+      options: { username: "testuser" },
+      env: {},
+      pageConfig: {},
+      oauth: null,
+    } as any);
+
+    expect(result).toEqual(sample);
+    expect(global.fetch).toHaveBeenCalledWith("https://mastodon.social/api/v1/accounts/1/statuses");
+  });
+
+  it("should throw UnauthorizedError on unauthorized response", async () => {
+    (global.fetch as any).mockResolvedValueOnce({
+      ok: false,
+      status: 401,
+    });
+
+    vi.mocked(fetchMastodonAccount).mockResolvedValueOnce(accountSample);
+
+    await expect(
+      fetchMastodonStatus({
+        options: { username: "testuser" },
+        env: {},
+        pageConfig: {},
+        oauth: null,
+      } as any),
+    ).rejects.toThrow(UnauthorizedError);
+  });
+
+  it("should throw error on invalid response data", async () => {
+    const invalidResponse = [{ invalid: "data" }];
+
+    (global.fetch as any).mockResolvedValueOnce({
+      ok: true,
+      json: () => Promise.resolve(invalidResponse),
+    });
+
+    await expect(
+      fetchMastodonStatus({
+        options: { username: "testuser" },
+        env: {},
+        pageConfig: {},
+        oauth: null,
+      } as any),
+    ).rejects.toThrow();
+  });
+});

package/src/shared/datasources/external/meta/oauth/config.ts

@@ -0,0 +1,16 @@
+import { Type, type Static, type StaticDecode } from "@sinclair/typebox";
+import { buildOAuthConfigSchema } from "~/shared/oauth";
+
+const metaOAuthConfig = Type.Object({
+  type: Type.Union([Type.Literal("short-lived"), Type.Literal("long-lived")]),
+  accessToken: Type.String(),
+  userId: Type.String(),
+  permissions: Type.Array(Type.String()),
+  expiresIn: Type.Number(),
+  tokenType: Type.String(),
+});
+
+export type MetaOAuthConfig = Static<typeof metaOAuthConfig>;
+
+const metaFullOAuthConfig = buildOAuthConfigSchema(metaOAuthConfig);
+export type MetaFullOAuthConfig = StaticDecode<typeof metaFullOAuthConfig>;

package/src/shared/datasources/external/meta/options.ts

@@ -0,0 +1,7 @@
+import { Type, type Static } from "@sinclair/typebox";
+
+export const metaOptions = Type.Object({
+  limit: Type.Optional(Type.Number()),
+});
+
+export type MetaOptions = Static<typeof metaOptions>;

package/src/shared/datasources/external/rss/fetcher.ts

@@ -0,0 +1,31 @@
+import type { RssOptions } from "./options";
+import { parseFeed } from "@rowanmanning/feed-parser";
+import type { RssSchema } from "./schema";
+import type { DatasourceFetcher } from "../../fetcher";
+import { createPlaceholderReplacer, placeholderRx } from "../../utils";
+
+const fetchRss: DatasourceFetcher<RssSchema, null, RssOptions> = async ({
+  options,
+  pageAttributes: attr,
+}) => {
+  const replacer = createPlaceholderReplacer(attr);
+  const url = options.url.replace(placeholderRx, replacer);
+  const content = await (await fetch(url)).text();
+  const feed = parseFeed(content);
+
+  const newFeed: RssSchema =
+    feed?.items.map((item) => ({
+      ...item,
+      pubDate: item.published ? item.published.toISOString() : new Date().toISOString(),
+    })) ?? [];
+
+  // const isValid = ajv.validate<RssSchema>(rssSchema, newFeed);
+
+  // if (!isValid) {
+  //   throw new Error(`fetchRss Error: Invalid Feed data (${url}): ${serializeAjvErrors(ajv.errors)}`);
+  // }
+
+  return newFeed;
+};
+
+export default fetchRss;

package/src/shared/datasources/external/rss/options.ts

@@ -0,0 +1,7 @@
+import { Type, type Static } from "@sinclair/typebox";
+
+export const rssOptions = Type.Object({
+  url: Type.String({ format: "uri" }),
+});
+
+export type RssOptions = Static<typeof rssOptions>;

package/src/shared/datasources/external/rss/sample.ts

@@ -0,0 +1,22 @@
+import type { RssSchema } from "./schema";
+
+export const sample = [
+  {
+    title: "Example title",
+    link: "https://example.com",
+    pubDate: "2022-01-01T00:00:00Z",
+    content: "Example content",
+  },
+  {
+    title: "Another example title",
+    link: "https://example.com",
+    pubDate: "2022-01-02T00:00:00Z",
+    content: "Another example content",
+  },
+  {
+    title: "Yet another example title",
+    link: "https://example.com",
+    pubDate: "2022-01-03T00:00:00Z",
+    content: "Yet another example content",
+  },
+] as RssSchema;

package/src/shared/datasources/external/rss/schema.ts

@@ -0,0 +1,42 @@
+import { Type, type Static } from "@sinclair/typebox";
+
+export const rssSchema = Type.Array(
+  Type.Object({
+    title: Type.Optional(
+      Type.String({
+        title: "Title",
+        description: "The title of the RSS feed item",
+      }),
+    ),
+    link: Type.Optional(
+      Type.String({
+        title: "Link",
+        description: "The link to the RSS feed item",
+      }),
+    ),
+    creator: Type.Optional(
+      Type.String({
+        title: "Creator",
+        description: "The creator of the RSS feed item",
+      }),
+    ),
+    content: Type.Optional(
+      Type.String({
+        title: "Content",
+        description: "The content of the RSS feed item",
+      }),
+    ),
+    pubDate: Type.Optional(
+      Type.String({
+        title: "Pub Date",
+        description: "The publication date of the RSS feed item",
+      }),
+    ),
+  }),
+  {
+    title: "RSS Feed",
+    description: "RSS feed items",
+  },
+);
+
+export type RssSchema = Static<typeof rssSchema>;

package/src/shared/datasources/external/threads/media/fetcher.ts

@@ -0,0 +1,53 @@
+import { type ThreadsMediaSchema, threadsMediaSchema } from "./schema";
+import type { MetaFullOAuthConfig } from "~/shared/datasources/external/meta/oauth/config";
+import invariant from "~/shared/utils/invariant";
+import { UnauthorizedError } from "~/shared/errors";
+import type { MetaOptions } from "~/shared/datasources/external/meta/options";
+import { stringifyObjectValues } from "~/shared/datasources/utils";
+import type { DatasourceFetcher } from "~/shared/datasources/fetcher";
+
+/**
+ * todo: add a way to retrieve media/posts from Threads for a given user other than "me"
+ */
+const fetchThreadsMediaDatasource: DatasourceFetcher<
+  ThreadsMediaSchema,
+  MetaFullOAuthConfig,
+  MetaOptions
+> = async ({ options, oauth }) => {
+  invariant(oauth?.config, "fetchThreadsMediaDatasource Error: OAuth config is required");
+
+  const params = new URLSearchParams({
+    ...stringifyObjectValues(options),
+    access_token: oauth.config.accessToken,
+    fields: [
+      "id",
+      "media_product_type",
+      "media_type",
+      "media_url",
+      "permalink",
+      "owner",
+      "username",
+      "text",
+      "timestamp",
+      "shortcode",
+      "thumbnail_url",
+      "children",
+      "is_quote_post",
+    ].join(","),
+  });
+
+  const response = await fetch(`https://graph.threads.net/v1.0/me/threads?${params.toString()}`);
+
+  if (!response.ok) {
+    if (response.status === 401) {
+      throw new UnauthorizedError(`fetchThreadsMediaDatasource Error: Unauthorized.`);
+    }
+    throw new Error(`fetchThreadsMediaDatasource Error: Response status: ${response.status}`);
+  }
+
+  const media = (await response.json()) as ThreadsMediaSchema;
+
+  return media;
+};
+
+export default fetchThreadsMediaDatasource;

package/src/shared/datasources/external/threads/media/sample.ts

@@ -0,0 +1,44 @@
+import type { ThreadsMediaSchema } from "./schema";
+
+export const sample = {
+  data: [
+    {
+      id: "1",
+      is_quote_post: false,
+      media_product_type: "THREADS",
+      media_type: "IMAGE",
+      media_url: "https://example.com/image.jpg",
+      owner: {
+        id: "1",
+      },
+      permalink: "https://example.com/post",
+      shortcode: "123456",
+      text: "Hello, world!",
+      thumbnail_url: "https://example.com/thumbnail.jpg",
+      timestamp: "2022-01-01T00:00:00Z",
+      username: "example",
+    },
+    {
+      id: "2",
+      is_quote_post: false,
+      media_product_type: "THREADS",
+      media_type: "VIDEO",
+      media_url: "https://example.com/video.mp4",
+      owner: {
+        id: "2",
+      },
+      permalink: "https://example.com/post",
+      shortcode: "654321",
+      text: "Goodbye, world!",
+      thumbnail_url: "https://example.com/thumbnail.jpg",
+      timestamp: "2022-01-02T00:00:00Z",
+      username: "example",
+    },
+  ],
+  paging: {
+    cursors: {
+      before: "",
+      after: "",
+    },
+  },
+} satisfies ThreadsMediaSchema;

package/src/shared/datasources/external/threads/media/schema.ts

@@ -0,0 +1,37 @@
+import { Type, type Static } from "@sinclair/typebox";
+
+export const threadsMediaSchema = Type.Object({
+  data: Type.Array(
+    Type.Object({
+      id: Type.String(),
+      media_product_type: Type.Literal("THREADS"),
+      media_type: Type.Union([
+        Type.Literal("TEXT_POST"),
+        Type.Literal("IMAGE"),
+        Type.Literal("VIDEO"),
+        Type.Literal("CAROUSEL_ALBUM"),
+        Type.Literal("AUDIO"),
+        Type.Literal("REPOST_FACADE"),
+      ]),
+      media_url: Type.String(),
+      permalink: Type.String(),
+      owner: Type.Object({
+        id: Type.String(),
+      }),
+      username: Type.String(),
+      text: Type.String(),
+      timestamp: Type.String(),
+      thumbnail_url: Type.String(),
+      shortcode: Type.String(),
+      is_quote_post: Type.Boolean(),
+    }),
+  ),
+  paging: Type.Object({
+    cursors: Type.Object({
+      before: Type.String(),
+      after: Type.String(),
+    }),
+  }),
+});
+
+export type ThreadsMediaSchema = Static<typeof threadsMediaSchema>;

package/src/shared/datasources/external/tiktok/oauth/config.ts

@@ -0,0 +1,17 @@
+import { Type, type StaticDecode, type Static } from "@sinclair/typebox";
+import { buildOAuthConfigSchema } from "~/shared/oauth";
+
+const tiktokOAuthConfig = Type.Object({
+  accessToken: Type.String(),
+  expiresIn: Type.Number(),
+  openId: Type.String(),
+  refreshExpiresIn: Type.Number(),
+  refreshToken: Type.String(),
+  scope: Type.String(),
+  tokenType: Type.String(),
+});
+
+export type TiktokOAuthConfig = Static<typeof tiktokOAuthConfig>;
+
+const tiktokFullOAuthConfig = buildOAuthConfigSchema(tiktokOAuthConfig);
+export type TiktokFullOAuthConfig = StaticDecode<typeof tiktokFullOAuthConfig>;

package/src/shared/datasources/external/tiktok/video/fetcher.ts

@@ -0,0 +1,39 @@
+import type { TiktokVideoOptions } from "./options";
+import type { TiktokVideoListSchema } from "./schema";
+import { UnauthorizedError } from "~/shared/errors";
+import type { TiktokFullOAuthConfig } from "../oauth/config";
+import type { DatasourceFetcher } from "~/shared/datasources/fetcher";
+
+const fetchTiktokVideoDatasource: DatasourceFetcher<
+  TiktokVideoListSchema,
+  TiktokFullOAuthConfig,
+  TiktokVideoOptions
+> = async ({ options, oauth }) => {
+  const params = new URLSearchParams({
+    access_token: oauth.config.accessToken,
+    fields: ["id", "title", "video_description", "duration", "cover_image_url", "embed_link"].join(","),
+  });
+
+  const url = `https://open.tiktokapis.com/v2/video/list/?${params.toString()}`;
+  const body = options;
+  const response = await fetch(url, {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+      Authorization: `Bearer ${oauth.config.accessToken}`,
+    },
+    body: JSON.stringify(body),
+  });
+
+  if (!response.ok) {
+    if (response.status === 401) {
+      throw new UnauthorizedError(`fetchTiktokVideoDatasource Error: Unauthorized.`);
+    }
+    throw new Error(`Response status: ${response.status}`);
+  }
+  const data = (await response.json()) as TiktokVideoListSchema;
+
+  return data;
+};
+
+export default fetchTiktokVideoDatasource;

package/src/shared/datasources/external/tiktok/video/options.ts

@@ -0,0 +1,8 @@
+import { Type, type Static } from "@sinclair/typebox";
+
+export const tiktokVideoOptions = Type.Object({
+  maxCount: Type.Optional(Type.Number()),
+  refreshInterval: Type.Optional(Type.Number()),
+});
+
+export type TiktokVideoOptions = Static<typeof tiktokVideoOptions>;

package/src/shared/datasources/external/tiktok/video/sample.ts

@@ -0,0 +1,26 @@
+import type { TiktokVideoListSchema } from "./schema";
+
+export const sample = {
+  data: {
+    has_more: true,
+    cursor: 1,
+    videos: [
+      {
+        title: "Example title",
+        video_description: "Example description",
+        cover_image_url: "https://example.com/image.jpg",
+        duration: 60,
+        embed_link: "https://example.com/embed",
+        id: "1",
+      },
+      {
+        title: "Another example title",
+        video_description: "Another example description",
+        cover_image_url: "https://example.com/image.jpg",
+        duration: 60,
+        embed_link: "https://example.com/embed",
+        id: "2",
+      },
+    ],
+  },
+} satisfies TiktokVideoListSchema;