@upstart.gg/sdk 0.0.95 → 0.0.97

package/src/shared/datasources/external/meta/oauth/config.ts

@@ -0,0 +1,16 @@
+import { Type, type Static, type StaticDecode } from "@sinclair/typebox";
+import { buildOAuthConfigSchema } from "~/shared/oauth";
+
+const metaOAuthConfig = Type.Object({
+  type: Type.Union([Type.Literal("short-lived"), Type.Literal("long-lived")]),
+  accessToken: Type.String(),
+  userId: Type.String(),
+  permissions: Type.Array(Type.String()),
+  expiresIn: Type.Number(),
+  tokenType: Type.String(),
+});
+
+export type MetaOAuthConfig = Static<typeof metaOAuthConfig>;
+
+const metaFullOAuthConfig = buildOAuthConfigSchema(metaOAuthConfig);
+export type MetaFullOAuthConfig = StaticDecode<typeof metaFullOAuthConfig>;

package/src/shared/datasources/external/meta/options.ts

@@ -0,0 +1,11 @@
+import { Type, type Static } from "@sinclair/typebox";
+import { providerOptions } from "../../provider-options";
+
+export const metaOptions = Type.Composite([
+  providerOptions,
+  Type.Object({
+    limit: Type.Optional(Type.Number()),
+  }),
+]);
+
+export type MetaOptions = Static<typeof metaOptions>;

package/src/shared/datasources/external/rss/fetcher.ts

@@ -0,0 +1,30 @@
+import type { RssOptions } from "./options";
+import { parseFeed } from "htmlparser2";
+import { type RssSchema, rssSchema } from "./schema";
+import { ajv, serializeAjvErrors } from "~/shared/ajv";
+import type { DatasourceFetcher } from "../../fetcher";
+import { createPlaceholderReplacer, placeholderRx } from "../../utils";
+
+const fetchRss: DatasourceFetcher<RssSchema, null, RssOptions> = async ({ options, attr }) => {
+  const replacer = createPlaceholderReplacer(attr);
+  const url = options.url.replace(placeholderRx, replacer);
+  const content = await (await fetch(url)).text();
+  const feed = parseFeed(content);
+
+  const newFeed: RssSchema =
+    feed?.items.map((item) => ({
+      ...item,
+      pubDate: item.pubDate ? item.pubDate.toISOString() : new Date().toISOString(),
+    })) ?? [];
+
+  const validate = ajv.compile<RssSchema>(rssSchema);
+  const isValid = validate(newFeed);
+
+  if (!isValid) {
+    throw new Error(`fetchRss Error: Invalid Feed data (${url}): ${serializeAjvErrors(validate.errors)}`);
+  }
+
+  return newFeed;
+};
+
+export default fetchRss;

package/src/shared/datasources/external/rss/options.ts

@@ -0,0 +1,11 @@
+import { Type, type Static } from "@sinclair/typebox";
+import { providerOptions } from "../../provider-options";
+
+export const rssOptions = Type.Composite([
+  providerOptions,
+  Type.Object({
+    url: Type.String({ format: "uri" }),
+  }),
+]);
+
+export type RssOptions = Static<typeof rssOptions>;

package/src/shared/datasources/external/rss/sample.ts

@@ -0,0 +1,22 @@
+import type { RssSchema } from "./schema";
+
+export const sample = [
+  {
+    title: "Example title",
+    link: "https://example.com",
+    pubDate: "2022-01-01T00:00:00Z",
+    content: "Example content",
+  },
+  {
+    title: "Another example title",
+    link: "https://example.com",
+    pubDate: "2022-01-02T00:00:00Z",
+    content: "Another example content",
+  },
+  {
+    title: "Yet another example title",
+    link: "https://example.com",
+    pubDate: "2022-01-03T00:00:00Z",
+    content: "Yet another example content",
+  },
+] as RssSchema;

package/src/shared/datasources/external/rss/schema.ts

@@ -0,0 +1,42 @@
+import { Type, type Static } from "@sinclair/typebox";
+
+export const rssSchema = Type.Array(
+  Type.Object({
+    title: Type.Optional(
+      Type.String({
+        title: "Title",
+        description: "The title of the RSS feed item",
+      }),
+    ),
+    link: Type.Optional(
+      Type.String({
+        title: "Link",
+        description: "The link to the RSS feed item",
+      }),
+    ),
+    creator: Type.Optional(
+      Type.String({
+        title: "Creator",
+        description: "The creator of the RSS feed item",
+      }),
+    ),
+    content: Type.Optional(
+      Type.String({
+        title: "Content",
+        description: "The content of the RSS feed item",
+      }),
+    ),
+    pubDate: Type.Optional(
+      Type.String({
+        title: "Pub Date",
+        description: "The publication date of the RSS feed item",
+      }),
+    ),
+  }),
+  {
+    title: "RSS Feed",
+    description: "RSS feed items",
+  },
+);
+
+export type RssSchema = Static<typeof rssSchema>;

package/src/shared/datasources/external/threads/media/fetcher.ts

@@ -0,0 +1,63 @@
+import { type ThreadsMediaSchema, threadsMediaSchema } from "./schema";
+import type { MetaFullOAuthConfig } from "~/shared/datasources/external/meta/oauth/config";
+import invariant from "~/shared/utils/invariant";
+import { UnauthorizedError } from "~/shared/errors";
+import type { MetaOptions } from "~/shared/datasources/external/meta/options";
+import { stringifyObjectValues } from "~/shared/datasources/utils";
+import { ajv, serializeAjvErrors } from "~/shared/ajv";
+import type { DatasourceFetcher } from "~/shared/datasources/fetcher";
+
+/**
+ * todo: add a way to retrieve media/posts from Threads for a given user other than "me"
+ */
+const fetchThreadsMediaDatasource: DatasourceFetcher<
+  ThreadsMediaSchema,
+  MetaFullOAuthConfig,
+  MetaOptions
+> = async ({ options, oauth }) => {
+  invariant(oauth?.config, "fetchThreadsMediaDatasource Error: OAuth config is required");
+
+  const params = new URLSearchParams({
+    ...stringifyObjectValues(options),
+    access_token: oauth.config.accessToken,
+    fields: [
+      "id",
+      "media_product_type",
+      "media_type",
+      "media_url",
+      "permalink",
+      "owner",
+      "username",
+      "text",
+      "timestamp",
+      "shortcode",
+      "thumbnail_url",
+      "children",
+      "is_quote_post",
+    ].join(","),
+  });
+
+  const response = await fetch(`https://graph.threads.net/v1.0/me/threads?${params.toString()}`);
+
+  if (!response.ok) {
+    if (response.status === 401) {
+      throw new UnauthorizedError(`fetchThreadsMediaDatasource Error: Unauthorized.`);
+    }
+    throw new Error(`fetchThreadsMediaDatasource Error: Response status: ${response.status}`);
+  }
+
+  const media = (await response.json()) as ThreadsMediaSchema;
+
+  const validate = ajv.compile<ThreadsMediaSchema>(threadsMediaSchema);
+  const isValid = validate(media);
+
+  if (!isValid) {
+    throw new Error(
+      `fetchThreadsMediaDatasource Error: Invalid Threads response data: ${serializeAjvErrors(validate.errors)}`,
+    );
+  }
+
+  return media;
+};
+
+export default fetchThreadsMediaDatasource;

package/src/shared/datasources/external/threads/media/sample.ts

@@ -0,0 +1,44 @@
+import type { ThreadsMediaSchema } from "./schema";
+
+export const sample = {
+  data: [
+    {
+      id: "1",
+      is_quote_post: false,
+      media_product_type: "THREADS",
+      media_type: "IMAGE",
+      media_url: "https://example.com/image.jpg",
+      owner: {
+        id: "1",
+      },
+      permalink: "https://example.com/post",
+      shortcode: "123456",
+      text: "Hello, world!",
+      thumbnail_url: "https://example.com/thumbnail.jpg",
+      timestamp: "2022-01-01T00:00:00Z",
+      username: "example",
+    },
+    {
+      id: "2",
+      is_quote_post: false,
+      media_product_type: "THREADS",
+      media_type: "VIDEO",
+      media_url: "https://example.com/video.mp4",
+      owner: {
+        id: "2",
+      },
+      permalink: "https://example.com/post",
+      shortcode: "654321",
+      text: "Goodbye, world!",
+      thumbnail_url: "https://example.com/thumbnail.jpg",
+      timestamp: "2022-01-02T00:00:00Z",
+      username: "example",
+    },
+  ],
+  paging: {
+    cursors: {
+      before: "",
+      after: "",
+    },
+  },
+} satisfies ThreadsMediaSchema;

package/src/shared/datasources/external/threads/media/schema.ts

@@ -0,0 +1,37 @@
+import { Type, type Static } from "@sinclair/typebox";
+
+export const threadsMediaSchema = Type.Object({
+  data: Type.Array(
+    Type.Object({
+      id: Type.String(),
+      media_product_type: Type.Literal("THREADS"),
+      media_type: Type.Union([
+        Type.Literal("TEXT_POST"),
+        Type.Literal("IMAGE"),
+        Type.Literal("VIDEO"),
+        Type.Literal("CAROUSEL_ALBUM"),
+        Type.Literal("AUDIO"),
+        Type.Literal("REPOST_FACADE"),
+      ]),
+      media_url: Type.String(),
+      permalink: Type.String(),
+      owner: Type.Object({
+        id: Type.String(),
+      }),
+      username: Type.String(),
+      text: Type.String(),
+      timestamp: Type.String(),
+      thumbnail_url: Type.String(),
+      shortcode: Type.String(),
+      is_quote_post: Type.Boolean(),
+    }),
+  ),
+  paging: Type.Object({
+    cursors: Type.Object({
+      before: Type.String(),
+      after: Type.String(),
+    }),
+  }),
+});
+
+export type ThreadsMediaSchema = Static<typeof threadsMediaSchema>;

package/src/shared/datasources/external/tiktok/oauth/config.ts

@@ -0,0 +1,17 @@
+import { Type, type StaticDecode, type Static } from "@sinclair/typebox";
+import { buildOAuthConfigSchema } from "~/shared/oauth";
+
+const tiktokOAuthConfig = Type.Object({
+  accessToken: Type.String(),
+  expiresIn: Type.Number(),
+  openId: Type.String(),
+  refreshExpiresIn: Type.Number(),
+  refreshToken: Type.String(),
+  scope: Type.String(),
+  tokenType: Type.String(),
+});
+
+export type TiktokOAuthConfig = Static<typeof tiktokOAuthConfig>;
+
+const tiktokFullOAuthConfig = buildOAuthConfigSchema(tiktokOAuthConfig);
+export type TiktokFullOAuthConfig = StaticDecode<typeof tiktokFullOAuthConfig>;

package/src/shared/datasources/external/tiktok/video/fetcher.ts

@@ -0,0 +1,49 @@
+import type { TiktokVideoOptions } from "./options";
+import { type TiktokVideoListSchema, tiktokVideoListSchema } from "./schema";
+import { UnauthorizedError } from "~/shared/errors";
+import { ajv, serializeAjvErrors } from "~/shared/ajv";
+import type { TiktokFullOAuthConfig } from "../oauth/config";
+import type { DatasourceFetcher } from "~/shared/datasources/fetcher";
+
+const fetchTiktokVideoDatasource: DatasourceFetcher<
+  TiktokVideoListSchema,
+  TiktokFullOAuthConfig,
+  TiktokVideoOptions
+> = async ({ options, oauth }) => {
+  const params = new URLSearchParams({
+    access_token: oauth.config.accessToken,
+    fields: ["id", "title", "video_description", "duration", "cover_image_url", "embed_link"].join(","),
+  });
+
+  const url = `https://open.tiktokapis.com/v2/video/list/?${params.toString()}`;
+  const { refreshInterval, ...body } = options;
+  const response = await fetch(url, {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+      Authorization: `Bearer ${oauth.config.accessToken}`,
+    },
+    body: JSON.stringify(body),
+  });
+
+  if (!response.ok) {
+    if (response.status === 401) {
+      throw new UnauthorizedError(`fetchTiktokVideoDatasource Error: Unauthorized.`);
+    }
+    throw new Error(`Response status: ${response.status}`);
+  }
+  const data = (await response.json()) as TiktokVideoListSchema;
+
+  const validate = ajv.compile<TiktokVideoListSchema>(tiktokVideoListSchema);
+  const isValid = validate(data);
+
+  if (!isValid) {
+    throw new Error(
+      `fetchTiktokVideoDatasource Error: Invalid TikTok response data: ${serializeAjvErrors(validate.errors)}`,
+    );
+  }
+
+  return data;
+};
+
+export default fetchTiktokVideoDatasource;

package/src/shared/datasources/external/tiktok/video/options.ts

@@ -0,0 +1,12 @@
+import { Type, type Static } from "@sinclair/typebox";
+import { providerOptions } from "~/shared/datasources/provider-options";
+
+export const tiktokVideoOptions = Type.Composite([
+  providerOptions,
+  Type.Object({
+    maxCount: Type.Optional(Type.Number()),
+    refreshInterval: Type.Optional(Type.Number()),
+  }),
+]);
+
+export type TiktokVideoOptions = Static<typeof tiktokVideoOptions>;

package/src/shared/datasources/external/tiktok/video/sample.ts

@@ -0,0 +1,26 @@
+import type { TiktokVideoListSchema } from "./schema";
+
+export const sample = {
+  data: {
+    has_more: true,
+    cursor: 1,
+    videos: [
+      {
+        title: "Example title",
+        video_description: "Example description",
+        cover_image_url: "https://example.com/image.jpg",
+        duration: 60,
+        embed_link: "https://example.com/embed",
+        id: "1",
+      },
+      {
+        title: "Another example title",
+        video_description: "Another example description",
+        cover_image_url: "https://example.com/image.jpg",
+        duration: 60,
+        embed_link: "https://example.com/embed",
+        id: "2",
+      },
+    ],
+  },
+} satisfies TiktokVideoListSchema;

package/src/shared/datasources/external/tiktok/video/schema.ts

@@ -0,0 +1,27 @@
+import { Type, type Static } from "@sinclair/typebox";
+
+export const tiktokVideoListSchema = Type.Object({
+  data: Type.Object({
+    has_more: Type.Boolean(),
+    cursor: Type.Number(),
+    videos: Type.Array(
+      Type.Object({
+        title: Type.String(),
+        video_description: Type.String(),
+        cover_image_url: Type.String(),
+        duration: Type.Number(),
+        embed_link: Type.String(),
+        id: Type.String(),
+      }),
+    ),
+  }),
+  error: Type.Optional(
+    Type.Object({
+      code: Type.String(),
+      message: Type.String(),
+      log_id: Type.String(),
+    }),
+  ),
+});
+
+export type TiktokVideoListSchema = Static<typeof tiktokVideoListSchema>;

package/src/shared/datasources/external/youtube/list/fetcher.ts

@@ -0,0 +1,46 @@
+import type { YoutubeListOptions } from "./options";
+import { type YoutubeListSchema, youtubeListSchema } from "./schema";
+import { UnauthorizedError } from "~/shared/errors";
+import { stringifyObjectValues } from "~/shared/datasources/utils";
+import { ajv, serializeAjvErrors } from "~/shared/ajv";
+import type { YoutubeFullOAuthConfig } from "../oauth/config";
+import type { DatasourceFetcher } from "~/shared/datasources/fetcher";
+
+const fetchYoutubeList: DatasourceFetcher<
+  YoutubeListSchema,
+  YoutubeFullOAuthConfig,
+  YoutubeListOptions
+> = async ({ options, oauth }) => {
+  const params = new URLSearchParams({
+    ...stringifyObjectValues(options),
+    part: "snippet,id",
+    type: "video",
+    videoEmbeddable: "true",
+    access_token: oauth.config.accessToken,
+  });
+
+  const url = `https://www.googleapis.com/youtube/v3/search?${params}`;
+
+  const response = await fetch(url);
+
+  if (!response.ok) {
+    if (response.status === 401) {
+      throw new UnauthorizedError(`fetchYoutubeList Error: Unauthorized.`);
+    }
+    throw new Error(`fetchYoutubeList Error: Response status: ${response.status}`);
+  }
+
+  const data = (await response.json()) as YoutubeListSchema;
+  const validate = ajv.compile<YoutubeListSchema>(youtubeListSchema);
+  const isValid = validate(data);
+
+  if (!isValid) {
+    throw new Error(
+      `fetchYoutubeList Error: Invalid Youtube response data: ${serializeAjvErrors(validate.errors)}`,
+    );
+  }
+
+  return data;
+};
+
+export default fetchYoutubeList;

package/src/shared/datasources/external/youtube/list/options.ts

@@ -0,0 +1,15 @@
+import { Type, type Static } from "@sinclair/typebox";
+import { providerOptions } from "~/shared/datasources/provider-options";
+
+export const youtubeListOptions = Type.Composite([
+  providerOptions,
+  Type.Object({
+    channelId: Type.String(),
+    order: Type.Optional(Type.String()),
+    maxResults: Type.Optional(Type.Number()),
+    regionCode: Type.Optional(Type.String()),
+    relevanceLanguage: Type.Optional(Type.String()),
+  }),
+]);
+
+export type YoutubeListOptions = Static<typeof youtubeListOptions>;

package/src/shared/datasources/external/youtube/list/sample.ts

@@ -0,0 +1,33 @@
+import type { YoutubeListSchema } from "./schema";
+
+export const sample = [
+  {
+    etag: "123456789",
+    id: {
+      videoId: "dQw4w9WgXcQ",
+      channelId: "UCXuqSBlHAE6Xw-yeJA0Tunw",
+      playlistId: "PLFgquLnL59alCl_2TQvOiD5Vgm1hCaGSI",
+    },
+    snippet: {
+      channelId: "UCXuqSBlHAE6Xw-yeJA0Tunw",
+      channelTitle: "RickAstleyVEVO",
+      description:
+        "Rick Astley's official music video for “Never Gonna Give You Up” Listen to Rick Astley: https://RickAstley.lnk.to/_listenYD Subscribe to the official Rick Astley YouTube channel: https://RickAstley.lnk.to/_subscribeYD Follow Rick Astley: Facebook: https://RickAstley.lnk.to/_followYD Twitter: https://RickAstley.lnk.to/_followYD Instagram: https://RickAstley.lnk.to/_followYD Website: https://RickAstley.lnk.to/_followYD Spotify: https://RickAstley.lnk.to/_followYD Lyrics: We're no strangers to love You know the rules and so do I A full commitment's what I'm thinking of You wouldn't get this from any other guy I just wanna tell you how I'm feeling Gotta make you understand Never gonna give you up Never gonna let you down Never gonna run around and desert you Never gonna make you cry Never gonna say goodbye Never gonna tell a lie and hurt you #RickAstley #NeverGonnaGiveYouUp #DancePop",
+      liveBroadcastContent: "none",
+      publishedAt: "2009-10-25",
+      thumbnails: {
+        default: {
+          height: 90,
+          url: "https://i.ytimg.com/vi/dQw4w9WgXcQ/default.jpg",
+          width: 120,
+        },
+        standard: {
+          height: 480,
+          url: "https://i.ytimg.com/vi/dQw4w9WgXcQ/sddefault.jpg",
+          width: 640,
+        },
+      },
+      title: "Rick Astley - Never Gonna Give You Up (Official Music Video)",
+    },
+  },
+] satisfies YoutubeListSchema;

package/src/shared/datasources/external/youtube/list/schema.ts

@@ -0,0 +1,38 @@
+import { Type, type Static } from "@sinclair/typebox";
+
+export const youtubeListSchema = Type.Array(
+  Type.Object({
+    etag: Type.String(),
+    id: Type.Object({
+      videoId: Type.String(),
+      channelId: Type.String(),
+      playlistId: Type.String(),
+    }),
+    snippet: Type.Object({
+      publishedAt: Type.String(),
+      channelId: Type.String(),
+      title: Type.String(),
+      description: Type.String(),
+      thumbnails: Type.Object({
+        default: Type.Object({
+          url: Type.String(),
+          width: Type.Number(),
+          height: Type.Number(),
+        }),
+        standard: Type.Object({
+          url: Type.String(),
+          width: Type.Number(),
+          height: Type.Number(),
+        }),
+      }),
+      channelTitle: Type.String(),
+      liveBroadcastContent: Type.String(),
+    }),
+  }),
+  {
+    title: "Youtube list schema",
+    description: "Schema for a list of Youtube videos",
+  },
+);
+
+export type YoutubeListSchema = Static<typeof youtubeListSchema>;

package/src/shared/datasources/external/youtube/oauth/config.ts

@@ -0,0 +1,15 @@
+import { Type, type Static } from "@sinclair/typebox";
+import { buildOAuthConfigSchema } from "~/shared/oauth";
+
+const youtubeOAuthConfig = Type.Object({
+  accessToken: Type.String(),
+  expiresIn: Type.Number(),
+  openId: Type.String(),
+  refreshExpiresIn: Type.Number(),
+  refreshToken: Type.String(),
+  scope: Type.String(),
+  tokenType: Type.String(),
+});
+
+const youtubeFullOAuthConfig = buildOAuthConfigSchema(youtubeOAuthConfig);
+export type YoutubeFullOAuthConfig = Static<typeof youtubeFullOAuthConfig>;

package/src/shared/datasources/fetcher.ts

@@ -0,0 +1,17 @@
+import type { Attributes } from "../attributes";
+import type { ProviderOptions } from "./provider-options";
+
+export type DatasourceFetcherParams<
+  OAuthProps = unknown,
+  Opts extends Record<string, unknown> = ProviderOptions,
+> = {
+  options: Opts;
+  attr: Attributes;
+  oauth: OAuthProps;
+};
+
+export type DatasourceFetcher<
+  T = unknown,
+  OAuthOpts = unknown,
+  Opts extends Record<string, unknown> = ProviderOptions,
+> = (params: DatasourceFetcherParams<OAuthOpts, Opts>) => Promise<T>;

package/src/shared/datasources/internal/blog/schema.ts

@@ -0,0 +1,69 @@
+import { Type, type Static } from "@sinclair/typebox";
+
+export const blogSchema = Type.Array(
+  Type.Object({
+    title: Type.String({
+      title: "Title",
+      description: "Blog post title",
+    }),
+    excerpt: Type.String({
+      title: "Excerpt",
+      description: "Short summary of the blog post",
+      format: "markdown",
+    }),
+    image: Type.Optional(
+      Type.String({
+        title: "Image URL",
+        format: "uri",
+        description: "Blog post image",
+      }),
+    ),
+    content: Type.String({
+      title: "Content",
+      description: "Blog post content",
+      format: "markdown",
+    }),
+    author: Type.Object({
+      name: Type.String({
+        title: "Author Name",
+        description: "Author's name",
+      }),
+    }),
+    publishedAt: Type.String({
+      title: "Published Date",
+      format: "date",
+      description: "Publication date in ISO format",
+    }),
+    slug: Type.String({
+      title: "Slug",
+      pattern: "^[a-z0-9]+(?:-[a-z0-9]+)*$",
+      description: "URL-friendly version of the title",
+    }),
+    status: Type.Union([Type.Literal("draft"), Type.Literal("published"), Type.Literal("archived")], {
+      title: "Status",
+      description: "Publication status of the blog post",
+    }),
+    categories: Type.Optional(
+      Type.Array(
+        Type.String({
+          title: "Categories",
+          description: "Blog post categories",
+        }),
+      ),
+    ),
+    tags: Type.Optional(
+      Type.Array(
+        Type.String({
+          title: "Tags",
+          description: "Blog post tags",
+        }),
+      ),
+    ),
+  }),
+  {
+    title: "Blog Posts",
+    description: "List of blog posts",
+  },
+);
+
+export type BlogSchema = Static<typeof blogSchema>;