@upstart.gg/sdk 0.0.96 → 0.0.97

package/src/shared/datarecords/external/airtable/handler.ts

@@ -0,0 +1,21 @@
+import type { AirtableOptions } from "./options";
+
+/**
+ * WARNING: not tested yet!!
+ *
+ * @todo test this function in a real environment
+ */
+export default async function airtableHandler(formData: FormData, options: AirtableOptions) {
+  const result = await fetch(
+    `https://api.airtable.com/v0/${options.baseId}/${encodeURIComponent(options.tableIdOrName)}`,
+    {
+      method: "POST",
+      body: JSON.stringify(formData),
+      headers: {
+        Authorization: `Bearer ${options.accessToken}`,
+        "Content-Type": "application/json",
+      },
+    },
+  );
+  return result.ok;
+}

package/src/shared/datarecords/external/airtable/options.ts

@@ -0,0 +1,22 @@
+import { Type, type Static } from "@sinclair/typebox";
+
+export const airtableOptions = Type.Object({
+  accessToken: Type.String({
+    description: "Airtable Personal Token or OAuth Access Token",
+  }),
+  baseId: Type.String({
+    pattern: "^app[A-Za-z0-9]+$",
+    description: 'Airtable Base ID starting with "app"',
+  }),
+  tableIdOrName: Type.Union([
+    Type.String({
+      pattern: "^tbl[A-Za-z0-9]+$",
+      description: 'Airtable Table ID starting with "tbl"',
+    }),
+    Type.String({
+      description: "Table name as shown in Airtable interface",
+    }),
+  ]),
+});
+
+export type AirtableOptions = Static<typeof airtableOptions>;

package/src/shared/datarecords/external/generic-webhook/handler.ts

@@ -0,0 +1,10 @@
+import type { GenericWebhookOptions } from "./options";
+
+export default async function genericWebhookHandler(formData: FormData, options: GenericWebhookOptions) {
+  const result = await fetch(options.url, {
+    method: "POST",
+    body: JSON.stringify(formData),
+    headers: options.headers,
+  });
+  return result.ok;
+}

package/src/shared/datarecords/external/generic-webhook/options.ts

@@ -0,0 +1,13 @@
+import { Type, type Static } from "@sinclair/typebox";
+
+export const genericWebhookOptions = Type.Object({
+  url: Type.String({ format: "uri", title: "Webhook URL" }),
+  headers: Type.Optional(
+    Type.Record(Type.String(), Type.String(), {
+      title: "Headers",
+      description: "Additional headers to include in the webhook request",
+    }),
+  ),
+});
+
+export type GenericWebhookOptions = Static<typeof genericWebhookOptions>;

package/src/shared/datarecords/external/google/oauth/config.ts

@@ -0,0 +1,30 @@
+import { Type, type Static } from "@sinclair/typebox";
+
+const googleOAuthTokenSchema = Type.Object({
+  access_token: Type.String(),
+  token_type: Type.String(), // Usually "Bearer"
+  expires_in: Type.Number(), // Seconds until token expires, typically 3600 (1 hour)
+  refresh_token: Type.Optional(Type.String()), // Only present in first OAuth exchange
+  scope: Type.String(), // Space-separated list of granted scopes
+  id_token: Type.Optional(Type.String()), // JWT token containing user info, if requested
+});
+
+export type GoogleOAuthToken = Static<typeof googleOAuthTokenSchema>;
+
+// You might also want to define the decoded id_token structure:
+const googleIdTokenSchema = Type.Object({
+  iss: Type.String(), // Issuer (usually 'https://accounts.google.com')
+  sub: Type.String(), // Unique Google ID for the user
+  aud: Type.String(), // Your client ID
+  iat: Type.Number(), // Issued at (timestamp)
+  exp: Type.Number(), // Expiration time (timestamp)
+  email: Type.Optional(Type.String()),
+  email_verified: Type.Optional(Type.Boolean()),
+  name: Type.Optional(Type.String()),
+  picture: Type.Optional(Type.String()),
+  given_name: Type.Optional(Type.String()),
+  family_name: Type.Optional(Type.String()),
+  locale: Type.Optional(Type.String()),
+});
+
+export type GoogleIdToken = Static<typeof googleIdTokenSchema>;

package/src/shared/datarecords/external/google/sheets/handler.ts

@@ -0,0 +1,26 @@
+import type { GoogleSheetsOptions } from "./options";
+
+/**
+ * WARNING: not tested yet!!
+ *
+ * @todo test this function in a real environment
+ */
+export default async function googleSheetsHandler(
+  formData: FormData,
+  options: GoogleSheetsOptions,
+  accessToken: string,
+) {
+  const url = `https://sheets.googleapis.com/v4/spreadsheets/${options.spreadsheetId}/values/A:Z:append?valueInputOption=RAW`;
+
+  const result = await fetch(url, {
+    method: "POST",
+    body: JSON.stringify({
+      values: [formData], // Wrap the row values in an array since API expects 2D array
+    }),
+    headers: {
+      Authorization: `Bearer ${accessToken}`,
+      "Content-Type": "application/json",
+    },
+  });
+  return result.ok;
+}

package/src/shared/datarecords/external/google/sheets/options.ts

@@ -0,0 +1,9 @@
+import { Type, type Static } from "@sinclair/typebox";
+
+export const googleSheetsOptions = Type.Object({
+  spreadsheetId: Type.String(),
+  // If targeting specific sheet
+  sheetName: Type.Optional(Type.String()),
+});
+
+export type GoogleSheetsOptions = Static<typeof googleSheetsOptions>;

package/src/shared/datarecords/types.ts

@@ -0,0 +1,120 @@
+import { Type, type Static } from "@sinclair/typebox";
+import { airtableOptions } from "./external/airtable/options";
+import { googleSheetsOptions } from "./external/google/sheets/options";
+import { genericWebhookOptions } from "./external/generic-webhook/options";
+
+export const connectorSchema = Type.Union([
+  Type.Literal("airtable"),
+  Type.Literal("google-sheets"),
+  // a generic webhook
+  Type.Literal("generic-webhook"),
+  // saved to Upstart platform
+  Type.Literal("internal"),
+]);
+
+export type DatarecordConnector = Static<typeof connectorSchema>;
+
+const internalDatarecord = Type.Object(
+  {
+    provider: Type.Literal("internal"),
+    // options: Type.Optional(Type.Any()),
+    schema: Type.Any({
+      title: "Schema",
+      description:
+        "JSON Schema of the datarecord. Always of type 'object' and representing a row that will be saved.",
+      examples: [
+        {
+          type: "object",
+          properties: {
+            firstname: { type: "string", title: "Firstname" },
+            lastname: { type: "string", title: "Lastname" },
+            email: { type: "string", format: "email", title: "Email" },
+          },
+          required: ["email"],
+          title: "Newsletter Subscription",
+        },
+      ],
+    }),
+    indexes: Type.Array(
+      Type.Object({
+        name: Type.String({ title: "Index name" }),
+        fields: Type.Array(Type.String(), { title: "Fields to index" }),
+        unique: Type.Optional(Type.Boolean({ title: "Unique index", default: false })),
+      }),
+      {
+        title: "Indexes",
+        description:
+          "IMPORTANT: Indexes to create on the datarecord. use it to enforce uniqueness or improve query performance.",
+      },
+    ),
+  },
+  {
+    examples: [
+      {
+        provider: "internal",
+        schema: {
+          type: "object",
+          properties: {
+            firstname: { type: "string", title: "Firstname" },
+            lastname: { type: "string", title: "Lastname" },
+            email: { type: "string", format: "email", title: "Email" },
+          },
+          required: ["email"],
+          title: "Newsletter Subscription",
+        },
+        indexes: [
+          {
+            name: "email_index",
+            fields: ["email"],
+            unique: true,
+          },
+        ],
+      },
+    ],
+  },
+);
+
+export const datarecordsConnectors = Type.Union([
+  Type.Object({
+    provider: Type.Literal("airtable"),
+    options: airtableOptions,
+  }),
+  Type.Object({
+    provider: Type.Literal("google-sheets"),
+    options: googleSheetsOptions,
+  }),
+  Type.Object({
+    provider: Type.Literal("generic-webhook"),
+    options: genericWebhookOptions,
+  }),
+  internalDatarecord,
+]);
+
+const datarecordMetadata = Type.Object({
+  id: Type.String({
+    title: "Datarecord ID",
+    comment: "A unique identifier for the datarecord, e.g., 'newsletter_subscriptions'",
+  }),
+  label: Type.String({
+    title: "Name of the datarecord",
+    comment: "For example, 'Newsletter Subscriptions'",
+  }),
+  description: Type.Optional(Type.String({ title: "Description of the datarecord" })),
+});
+
+const datarecordManifest = Type.Composite([datarecordsConnectors, datarecordMetadata]);
+
+export const internalDatarecordManifest = Type.Composite([datarecordMetadata, internalDatarecord]);
+
+export type DatarecordManifest = Static<typeof datarecordManifest>;
+
+export const datarecordsMap = Type.Record(Type.String(), datarecordManifest, {
+  title: "Datarecords map",
+  description: "The map of Datarecords available",
+});
+
+export type DatarecordsMap = Static<typeof datarecordsMap>;
+
+export type DatarecordResolved<T extends DatarecordsMap> = {
+  [K in keyof T]: unknown;
+};

package/src/shared/datarecords.ts

@@ -0,0 +1,5 @@
+import type { DatarecordsMap } from "./datarecords/types";
+
+export function defineDataRecords<T extends DatarecordsMap>(datarecords: T) {
+  return datarecords;
+}

package/src/shared/datasources/README.md

@@ -0,0 +1,3 @@
+# Data Sources
+
+This directory contains the data sources nativelty supported by the SDK.

package/src/shared/datasources/external/facebook/posts/fetcher.ts

@@ -0,0 +1,62 @@
+import { facebookPostSchema, type FacebookPostSchema } from "./schema";
+import type { MetaFullOAuthConfig } from "../../meta/oauth/config";
+import { UnauthorizedError } from "~/shared/errors";
+import type { MetaOptions } from "../../meta/options";
+import { stringifyObjectValues } from "../../../utils";
+import { ajv, serializeAjvErrors } from "~/shared/ajv";
+import type { DatasourceFetcher } from "~/shared/datasources/fetcher";
+
+const fetchFacebookPostDatasource: DatasourceFetcher<
+  FacebookPostSchema,
+  MetaFullOAuthConfig,
+  MetaOptions
+> = async ({ options, oauth }) => {
+  const params = new URLSearchParams({
+    ...stringifyObjectValues(options),
+    fields: [
+      "from",
+      "permalink_url",
+      "name",
+      "description",
+      "caption",
+      "id",
+      "is_hidden",
+      "message",
+      "application",
+      "object_id",
+      "link",
+      "is_published",
+      "properties",
+      "status_type",
+      "story",
+      "type",
+      "actions",
+      "call_to_action",
+      "child_attachments",
+    ].join(","),
+    access_token: oauth.config.accessToken,
+  });
+
+  const response = await fetch(`https://graph.facebook.com/me/posts?${params}`);
+
+  if (!response.ok) {
+    if (response.status === 401) {
+      throw new UnauthorizedError(`fetchFacebookPostDatasource Error: Unauthorized.`);
+    }
+    throw new Error(`fetchFacebookPostDatasource Error: Response status: ${response.status}`);
+  }
+
+  const post = (await response.json()) as FacebookPostSchema;
+
+  const validate = ajv.compile<FacebookPostSchema>(facebookPostSchema);
+
+  if (!validate(post)) {
+    throw new Error(
+      `fetchFacebookPostDatasource Error: Invalid JSON object: ${serializeAjvErrors(validate.errors)}`,
+    );
+  }
+
+  return post;
+};
+
+export default fetchFacebookPostDatasource;

package/src/shared/datasources/external/facebook/posts/sample.ts

@@ -0,0 +1,35 @@
+import type { FacebookPostSchema } from "./schema";
+
+export const sample = {
+  data: [
+    {
+      from: {
+        name: "Facebook",
+        id: "20531316728",
+      },
+      id: "20531316728_10154052815206729",
+      permalink_url: "https://www.facebook.com/facebook/posts/10154052815206729",
+      is_hidden: false,
+      message: "Great photo!",
+      object_id: "10154052815196729",
+      link: "https://www.facebook.com/photo.php?fbid=10154052815196729&set=a.10150278999681729.345701.20531316728&type=3",
+      is_published: true,
+      status_type: "added_photos",
+      type: "photo",
+      actions: [
+        {
+          name: "Comment",
+          link: "https://www.facebook.com/20531316728/posts/10154052815206729",
+        },
+        {
+          name: "Like",
+          link: "https://www.facebook.com/20531316728/posts/10154052815206729",
+        },
+      ],
+    },
+  ],
+  paging: {
+    next: "url",
+    previous: "url",
+  },
+} satisfies FacebookPostSchema;

package/src/shared/datasources/external/facebook/posts/schema.ts

@@ -0,0 +1,33 @@
+import { Type, type Static } from "@sinclair/typebox";
+
+export const facebookPostSchema = Type.Object({
+  data: Type.Array(
+    Type.Object({
+      from: Type.Object({
+        name: Type.String(),
+        id: Type.String(),
+      }),
+      id: Type.String(),
+      permalink_url: Type.String(),
+      is_hidden: Type.Boolean(),
+      message: Type.Optional(Type.String()),
+      object_id: Type.String(),
+      link: Type.String(),
+      is_published: Type.Boolean(),
+      status_type: Type.String(),
+      type: Type.String(),
+      actions: Type.Array(
+        Type.Object({
+          name: Type.String(),
+          link: Type.String(),
+        }),
+      ),
+    }),
+  ),
+  paging: Type.Object({
+    previous: Type.Optional(Type.String()),
+    next: Type.Optional(Type.String()),
+  }),
+});
+
+export type FacebookPostSchema = Static<typeof facebookPostSchema>;

package/src/shared/datasources/external/facebook/posts/tests/fetcher.test.ts

@@ -0,0 +1,91 @@
+import { describe, it, expect, vi, beforeEach } from "vitest";
+import fetchFacebookPostDatasource from "../fetcher";
+import { UnauthorizedError } from "~/shared/errors";
+import type { MetaOAuthConfig } from "~/shared/datasources/external/meta/oauth/config";
+import type { MetaOptions } from "~/shared/datasources/external/meta/options";
+import type { DatasourceFetcherParams } from "~/shared/datasources/fetcher";
+
+// Mock the fetch function
+global.fetch = vi.fn();
+
+describe("fetchFacebookPostDatasource", () => {
+  beforeEach(() => {
+    vi.resetAllMocks();
+  });
+
+  it("should fetch Facebook posts successfully", async () => {
+    const mockResponse = {
+      data: [
+        {
+          object_id: "123",
+          id: "123",
+          message: "Test post",
+          from: { name: "Test User", id: "456" },
+          permalink_url: "https://facebook.com/post/123",
+          link: "https://facebook.com/post/123",
+          is_hidden: false,
+          is_published: true,
+          type: "status",
+          status_type: "mobile_status_update",
+          actions: [{ name: "Comment", link: "https://facebook.com/post/123" }],
+        },
+      ],
+      paging: { next: "https://graph.facebook.com/me/posts?after=123" },
+    };
+
+    (global.fetch as any).mockResolvedValueOnce({
+      ok: true,
+      json: () => Promise.resolve(mockResponse),
+    });
+
+    // @ts-ignore
+    const result = await fetchFacebookPostDatasource({
+      options: { limit: 10 },
+      oauth: { config: { accessToken: "test-token" } },
+      // @ts-ignore
+      attr: {
+        id: "me",
+        siteId: "123",
+      },
+    } as DatasourceFetcherParams<MetaOAuthConfig, MetaOptions>);
+
+    expect(result).toEqual(mockResponse);
+    expect(global.fetch).toHaveBeenCalledWith(
+      expect.stringContaining("https://graph.facebook.com/me/posts?"),
+    );
+  });
+
+  it("should throw UnauthorizedError on unauthorized response", async () => {
+    (global.fetch as any).mockResolvedValueOnce({
+      ok: false,
+      status: 401,
+    });
+
+    await expect(
+      // @ts-ignore
+      fetchFacebookPostDatasource({
+        options: { limit: 10 },
+        oauth: { config: { accessToken: "invalid-token" } },
+        attr: {},
+      } as DatasourceFetcherParams<MetaOAuthConfig, MetaOptions>),
+    ).rejects.toThrow(UnauthorizedError);
+  });
+
+  it("should throw error on invalid response data", async () => {
+    const invalidResponse = { invalid: "data" };
+
+    (global.fetch as any).mockResolvedValueOnce({
+      ok: true,
+      json: () => Promise.resolve(invalidResponse),
+    });
+
+    await expect(
+      // @ts-ignore
+      fetchFacebookPostDatasource({
+        options: { limit: 10 },
+        oauth: { config: { accessToken: "test-token" } },
+        attr: {},
+      } as DatasourceFetcherParams<MetaOAuthConfig, MetaOptions>),
+    ).rejects.toThrow();
+  });
+});

package/src/shared/datasources/external/http-json/fetcher.ts

@@ -0,0 +1,28 @@
+import type { DatasourceFetcher } from "../../fetcher";
+import { createPlaceholderReplacer, placeholderRx } from "../../utils";
+import type { HttpJsonOptions } from "./options";
+
+/**
+ * For this fetcher, validation is done outside of the fetcher.
+ */
+const fetchHttpJSON: DatasourceFetcher<unknown, null, HttpJsonOptions> = async ({ options, attr }) => {
+  const replacer = createPlaceholderReplacer(attr);
+  const url = options.url.replace(placeholderRx, replacer);
+  const headers: Record<string, string> = {};
+
+  if (options.headers) {
+    for (const [key, value] of Object.entries(options.headers ?? {})) {
+      headers[key] = (value as string).replace(placeholderRx, replacer);
+    }
+  }
+
+  const response = await fetch(url, { headers });
+
+  if (!response.ok) {
+    throw new Error(`fetchHttpJSON Error: Response status: ${response.status}`);
+  }
+
+  return response.json();
+};
+
+export default fetchHttpJSON;

package/src/shared/datasources/external/http-json/options.ts

@@ -0,0 +1,12 @@
+import { Type, type Static } from "@sinclair/typebox";
+import { providerOptions } from "../../provider-options";
+
+export const httpJsonOptions = Type.Composite([
+  providerOptions,
+  Type.Object({
+    url: Type.String({ format: "uri" }),
+    headers: Type.Optional(Type.Record(Type.String(), Type.String())),
+  }),
+]);
+
+export type HttpJsonOptions = Static<typeof httpJsonOptions>;

package/src/shared/datasources/external/http-json/schema.ts

@@ -0,0 +1,6 @@
+import { Type, type Static } from "@sinclair/typebox";
+
+export const jsonObjectSchema = Type.Object({}, { additionalProperties: true });
+export const jsonArraySchema = Type.Array(jsonObjectSchema, { title: "Http JSON" });
+
+export type JSONArraySchema = Static<typeof jsonArraySchema>;

package/src/shared/datasources/external/http-json/tests/fetcher.test.ts

@@ -0,0 +1,70 @@
+import { describe, it, expect, vi, beforeEach } from "vitest";
+import fetchHttpJSON from "../fetcher";
+import type { HttpJsonOptions } from "../options";
+import type { DatasourceFetcherParams } from "~/shared/datasources/fetcher";
+
+// Mock the fetch function
+global.fetch = vi.fn();
+
+describe("fetchHttpJSON", () => {
+  beforeEach(() => {
+    vi.resetAllMocks();
+  });
+
+  it("should fetch JSON data successfully", async () => {
+    const mockResponse = { data: "test" };
+
+    (global.fetch as any).mockResolvedValueOnce({
+      ok: true,
+      json: () => Promise.resolve(mockResponse),
+    });
+
+    const result = await fetchHttpJSON({
+      options: { url: "https://api.example.com/data" },
+      attr: {},
+      oauth: null,
+    } as DatasourceFetcherParams<null, HttpJsonOptions>);
+
+    expect(result).toEqual(mockResponse);
+    expect(global.fetch).toHaveBeenCalledWith("https://api.example.com/data", { headers: {} });
+  });
+
+  it("should replace placeholders in URL and headers", async () => {
+    const mockResponse = { data: "test" };
+
+    (global.fetch as any).mockResolvedValueOnce({
+      ok: true,
+      json: () => Promise.resolve(mockResponse),
+    });
+
+    const result = await fetchHttpJSON({
+      options: {
+        url: "https://api.example.com/{{dataType}}",
+        headers: { "X-API-Key": "{{apiKey}}" },
+      },
+      attr: { dataType: "users", apiKey: "secret-key" },
+      env: {},
+      oauth: null,
+    } as unknown as DatasourceFetcherParams<null, HttpJsonOptions>);
+
+    expect(result).toEqual(mockResponse);
+    expect(global.fetch).toHaveBeenCalledWith("https://api.example.com/users", {
+      headers: { "X-API-Key": "secret-key" },
+    });
+  });
+
+  it("should throw error on non-OK response", async () => {
+    (global.fetch as any).mockResolvedValueOnce({
+      ok: false,
+      status: 404,
+    });
+
+    await expect(
+      fetchHttpJSON({
+        options: { url: "https://api.example.com/notfound" },
+        attr: {},
+        oauth: null,
+      } as DatasourceFetcherParams<null, HttpJsonOptions>),
+    ).rejects.toThrow("fetchHttpJSON Error: Response status: 404");
+  });
+});

package/src/shared/datasources/external/instagram/feed/fetcher.ts

@@ -0,0 +1,43 @@
+import { instagramFeedSchema, type InstagramFeedSchema } from "./schema";
+import type { MetaFullOAuthConfig } from "~/shared/datasources/external/meta/oauth/config";
+import { UnauthorizedError } from "~/shared/errors";
+import type { MetaOptions } from "~/shared/datasources/external/meta/options";
+import { stringifyObjectValues } from "~/shared/datasources/utils";
+import { ajv, serializeAjvErrors } from "~/shared/ajv";
+import type { DatasourceFetcher } from "~/shared/datasources/fetcher";
+
+const fetchInstagramFeedDatasource: DatasourceFetcher<
+  InstagramFeedSchema,
+  MetaFullOAuthConfig,
+  MetaOptions
+> = async ({ options, oauth }) => {
+  const params = new URLSearchParams({
+    ...stringifyObjectValues(options),
+    access_token: oauth.config.accessToken,
+    fields: ["id", "caption", "timestamp", "thumbnail_url", "media_url", "permalink", "media_type"].join(","),
+  });
+
+  const response = await fetch(`https://graph.instagram.com/me/media?${params.toString()}`);
+
+  if (!response.ok) {
+    if (response.status === 401) {
+      throw new UnauthorizedError(`fetchInstagramFeedDatasource Error: Unauthorized.`);
+    }
+    throw new Error(`fetchInstagramFeedDatasource Error: Response status: ${response.status}`);
+  }
+
+  const feed = (await response.json()) as InstagramFeedSchema;
+
+  const validate = ajv.compile<InstagramFeedSchema>(instagramFeedSchema);
+  const isValid = validate(feed);
+
+  if (!isValid) {
+    throw new Error(
+      `fetchInstagramFeedDatasource Error: Invalid Instagram response data: ${serializeAjvErrors(validate.errors)}`,
+    );
+  }
+
+  return feed;
+};
+
+export default fetchInstagramFeedDatasource;