@upstart.gg/sdk 0.0.130 → 0.0.131

package/src/shared/datasources/types.ts

@@ -0,0 +1,490 @@
+import { Type, type Static } from "@sinclair/typebox";
+import { httpJsonOptions } from "./external/http-json/options";
+import { StringEnum } from "../utils/string-enum";
+import { toLLMSchema } from "../utils/llm";
+
+export const providersSchema = Type.Union([
+  // Type.Literal("facebook-posts"),
+  // Type.Literal("instagram-feed"),
+  // Type.Literal("mastodon-account"),
+  // Type.Literal("mastodon-status"),
+  // Type.Literal("mastodon-status-list"),
+  Type.Literal("internal"),
+  // Type.Literal("rss"),
+  // // Type.Literal("threads-media"),
+  // // Type.Literal("tiktok-video"),
+  // Type.Literal("youtube-list"),
+  // Type.Literal("http-json"),
+  // Type.Literal("internal-blog"),
+  // Type.Literal("internal-changelog"),
+  // // Type.Literal("internal-contact-info"),
+  // Type.Literal("internal-faq"),
+  // Type.Literal("internal-links"),
+  // Type.Literal("internal-recipes"),
+  // Type.Literal("internal-restaurant"),
+  // Type.Literal("internal-cv"),
+]);
+
+export type DatasourceProvider = Static<typeof providersSchema>;
+
+// const datasourceProviderManifest = Type.Composite([
+//   Type.Object({
+//     id: Type.String({
+//       title: "ID",
+//       description:
+//         "Unique identifier of the datasource. Used to reference the datasource in the system. Use a url-safe string like a slug.",
+//     }),
+//     label: Type.String({ title: "Label", description: "Label of the datasource displayed in the UI" }),
+//     schema: Type.Null({
+//       description: "Always null for provider datasources. The schema is defined by the provider.",
+//     }),
+//     ttlMinutes: Type.Optional(
+//       Type.Number({
+//         title: "Time to live",
+//         description:
+//           "Time to live in minutes. If set to -1, it never expires and has to be manually refreshed. If set to 0, the datasource is always fetched live. If > 0, then the datasource is feteched every N minutes.",
+//       }),
+//     ),
+//     refresh: Type.Optional(
+//       Type.Object(
+//         {
+//           method: Type.Union([Type.Literal("interval"), Type.Literal("manual"), Type.Literal("live")]),
+//           interval: Type.Optional(Type.Number()),
+//         },
+//         {
+//           title: "Refresh options",
+//           description: "Options to refresh the datasource",
+//         },
+//       ),
+//     ),
+//   }),
+// ]);
+
+// export type DatasourceProviderManifest = Static<typeof datasourceProviderManifest>;
+
+const datasourceInternalManifest = Type.Object(
+  {
+    id: Type.String({
+      title: "ID",
+      description:
+        "Unique identifier of the datasource. Used to reference the datasource in the system. Use a url-safe string like a slug.",
+    }),
+    label: Type.String({ title: "Label", description: "Label of the datasource displayed in the UI" }),
+    provider: Type.Literal("internal", {
+      title: "Internal",
+      description: "Internal datasource saved locally in Upstart.",
+    }),
+    schema: Type.Any({
+      title: "Schema",
+      description: "JSON Schema of datasource. MUST Always an array of objects.",
+    }),
+    indexes: Type.Optional(
+      Type.Array(
+        Type.Object({
+          name: Type.String({ title: "Index name" }),
+          fields: Type.Array(Type.String(), { title: "Fields to index" }),
+          unique: Type.Optional(Type.Boolean({ title: "Unique index", default: false })),
+        }),
+        {
+          title: "Indexes",
+          description:
+            "IMPORTANT: Indexes to create on the datasource. use it to enforce uniqueness or improve query performance.",
+        },
+      ),
+    ),
+  },
+  {
+    examples: [
+      {
+        id: "customers",
+        label: "Customers",
+        provider: "internal",
+        schema: {
+          type: "array",
+          items: {
+            type: "object",
+            properties: {
+              name: { type: "string", title: "Name" },
+              email: { type: "string", title: "Email", format: "email" },
+            },
+            required: ["name", "email"],
+            title: "Customer",
+            examples: [
+              { name: "John Doe", email: "john.doe@example.com" },
+              {
+                name: "Jane Smith",
+                email: "jane.smith@example.com",
+              },
+              {
+                name: "Alice Johnson",
+                email: "alice.johnson@example.com",
+              },
+              {
+                name: "Bob Brown",
+                email: "bob.brown@example.com",
+              },
+              {
+                name: "Charlie Davis",
+                email: "charlie.davis@example.com",
+              },
+            ],
+          },
+        },
+        indexes: [
+          {
+            name: "idx_customers_email",
+            fields: ["email"],
+            unique: true,
+          },
+        ],
+      },
+      {
+        id: "blog_posts",
+        label: "Blog Posts",
+        provider: "internal",
+        schema: {
+          type: "array",
+          items: {
+            type: "object",
+            properties: {
+              title: { type: "string", title: "Title" },
+              content: { type: "string", title: "Content" },
+              author: { type: "string", title: "Author" },
+            },
+            required: ["title", "content", "author"],
+            title: "Blog Post",
+            examples: [
+              {
+                title: "My First Blog Post",
+                content: "This is the content of my first blog post.",
+                author: "John Doe",
+              },
+              {
+                title: "Exploring the Cosmos",
+                content: "A journey through the stars and galaxies.",
+                author: "Jane Smith",
+              },
+              {
+                title: "The Art of Cooking",
+                content: "Delicious recipes and cooking tips.",
+                author: "Alice Johnson",
+              },
+              {
+                title: "Traveling the World",
+                content: "My adventures in different countries.",
+                author: "Bob Brown",
+              },
+              {
+                title: "Technology Trends",
+                content: "The latest trends in technology.",
+                author: "Charlie Davis",
+              },
+            ],
+          },
+        },
+        indexes: [
+          {
+            name: "idx_blog_posts_title",
+            fields: ["title"],
+            unique: true,
+          },
+        ],
+      },
+    ],
+  },
+);
+
+export type InternalDatasource = Static<typeof datasourceInternalManifest>;
+
+// const datasourceJsonManifest = Type.Composite([
+//   datasourceBaseFields,
+//   Type.Object({
+//     provider: Type.Literal("http-json", {
+//       title: "JSON Array",
+//       description: "JSON array datasource.",
+//     }),
+//     options: httpJsonOptions,
+//     schema: Type.Any({
+//       title: "Schema",
+//       description: "JSON Schema of datasource. Always an array of objects.",
+//       examples: [
+//         {
+//           type: "array",
+//           items: {
+//             type: "object",
+//             properties: {
+//               id: { type: "string", title: "ID" },
+//               title: { type: "string", title: "Title" },
+//               firstname: { type: "string", title: "Firstname" },
+//               lastname: { type: "string", title: "Lastname" },
+//               createdAt: { type: "string", format: "date-time", title: "Created at" },
+//               email: { type: "string", format: "email", title: "Email" },
+//             },
+//             required: ["id", "title", "firstname", "lastname", "email", "createdAt"],
+//             title: "Employee",
+//           },
+//           title: "Employees",
+//           description: "Employees list",
+//         },
+//       ],
+//     }),
+//   }),
+// ]);
+
+// type DatasourceJsonArrayManifest = Static<typeof datasourceJsonManifest>;
+
+// Fow now, let support only custom (internal) datasource
+// export const datasourceManifest = datasourceCustomManifest;
+export const datasourceManifest = datasourceInternalManifest;
+
+export function getDatasourceManifestForLLM() {
+  return toLLMSchema(datasourceManifest);
+}
+
+export type Datasource = Static<typeof datasourceManifest>;
+export const datasourcesList = Type.Array(datasourceManifest);
+export type DatasourcesList = Static<typeof datasourcesList>;
+
+const stringFilter = Type.Object({
+  field: Type.String(),
+  op: Type.Union([
+    Type.Literal("eq"),
+    Type.Literal("ne"),
+    Type.Literal("contains"),
+    Type.Literal("notContains"),
+    Type.Literal("startsWith"),
+    Type.Literal("notStartsWith"),
+    Type.Literal("endsWith"),
+    Type.Literal("notEndsWith"),
+  ]),
+  value: Type.String(),
+});
+
+const numberFilter = Type.Object({
+  field: Type.String(),
+  op: Type.Union([
+    Type.Literal("eq"),
+    Type.Literal("ne"),
+    Type.Literal("lt"),
+    Type.Literal("lte"),
+    Type.Literal("gt"),
+    Type.Literal("gte"),
+  ]),
+  value: Type.Number(),
+});
+
+const dateFilterAbsolute = Type.Object({
+  field: Type.String(),
+  op: Type.Union([Type.Literal("before"), Type.Literal("after")]),
+  value: Type.String(),
+});
+
+const dateFilterRelative = Type.Object({
+  field: Type.String(),
+  op: Type.Union([Type.Literal("beforeNow"), Type.Literal("afterNow")]),
+  value: Type.Null(),
+});
+
+const arrayFilter = Type.Object({
+  field: Type.String(),
+  op: Type.Union([
+    Type.Literal("contains"),
+    Type.Literal("notContains"),
+    Type.Literal("containsAll"),
+    Type.Literal("containsAny"),
+    Type.Literal("notContainsAny"),
+  ]),
+  value: Type.Array(Type.String()),
+});
+
+const booleanFilter = Type.Object({
+  field: Type.String(),
+  op: Type.Literal("eq"),
+  value: Type.Boolean(),
+});
+
+export const queryFilter = Type.Union([
+  stringFilter,
+  numberFilter,
+  dateFilterAbsolute,
+  dateFilterRelative,
+  arrayFilter,
+  booleanFilter,
+]);
+
+const filterExpression = Type.Recursive(
+  (This) =>
+    Type.Union(
+      [
+        Type.Object({
+          op: Type.Literal("and"),
+          fields: Type.Array(Type.Union([queryFilter, This]), { title: "Indexed Fields" }),
+        }),
+        Type.Object({
+          op: Type.Literal("or"),
+          fields: Type.Array(Type.Union([queryFilter, This])),
+        }),
+      ],
+      {
+        default: {
+          op: "and",
+          fields: [],
+        },
+        examples: [
+          {
+            op: "and",
+            fields: [
+              {
+                field: "$id",
+                op: "eq",
+                value: "123",
+              },
+              {
+                field: "$publicationDate",
+                op: "beforeNow",
+              },
+            ],
+          },
+        ],
+      },
+    ),
+  {
+    title: "Filter Expression",
+    description: "Expression used to filter query results. Can be a combination of and/or conditions.",
+  },
+);
+
+export const querySchema = Type.Object(
+  {
+    id: Type.String({
+      title: "Query ID",
+      description:
+        "Unique identifier for the query. Used to reference the query in the system. URL-safe string like a slug.",
+    }),
+    label: Type.String({
+      title: "Label",
+      maxLength: 100,
+      description: "Label of the query displayed in the UI",
+    }),
+    datasourceId: Type.String({
+      title: "Database",
+      description: "ID of the datasource to query",
+    }),
+    limit: Type.Number({
+      title: "Limit",
+      description: "Limit the number of records to fetch from the datasource.",
+      minimum: 1,
+      default: 10,
+    }),
+    sortDirection: Type.Optional(
+      StringEnum(["asc", "desc"], {
+        title: "Sort",
+        enumNames: ["Ascending", "Descending"],
+        description: "Direction to sort the records by",
+        default: "desc",
+      }),
+    ),
+    sortField: Type.Optional(
+      Type.String({
+        title: "Sort Field",
+        description: "Field to sort by (must be an indexed field)",
+        default: "$publicationDate",
+      }),
+    ),
+    filters: Type.Optional(filterExpression),
+    parameters: Type.Optional(
+      Type.Array(
+        Type.Object({
+          field: Type.String({ title: "Field", description: "Field name to use as parameter" }),
+          op: Type.Union(
+            [
+              Type.Literal("eq"),
+              Type.Literal("ne"),
+              Type.Literal("contains"),
+              Type.Literal("notContains"),
+              Type.Literal("startsWith"),
+              Type.Literal("notStartsWith"),
+              Type.Literal("endsWith"),
+              Type.Literal("notEndsWith"),
+              Type.Literal("lt"),
+              Type.Literal("lte"),
+              Type.Literal("gt"),
+              Type.Literal("gte"),
+              Type.Literal("before"),
+              Type.Literal("after"),
+              Type.Literal("beforeNow"),
+              Type.Literal("afterNow"),
+              Type.Literal("containsAll"),
+              Type.Literal("containsAny"),
+              Type.Literal("notContainsAny"),
+            ],
+            { title: "Operator", description: "Operator to use for the parameter" },
+          ),
+        }),
+        {
+          title: "Parameters",
+          description:
+            "Field name and operator that will be used as parameters when using the query in pages. Only indexed fields can be used as parameters.",
+          default: [],
+          examples: [
+            [{ field: "$slug", op: "eq" }],
+            [{ field: "category", op: "eq" }],
+            [{ field: "tags", op: "containsAny" }],
+            [{ field: "author", op: "eq" }],
+            [{ field: "title", op: "contains" }],
+          ],
+        },
+      ),
+    ),
+  },
+  {
+    examples: [
+      {
+        id: "latest-posts",
+        label: "Latest posts",
+        datasourceId: "blog_posts",
+        limit: 5,
+        sortDirection: "desc",
+        sortField: "$publicationDate",
+      },
+      {
+        id: "posts-by-category",
+        label: "Posts by category",
+        datasourceId: "blog_posts",
+        limit: 10,
+        sortDirection: "desc",
+        sortField: "$publicationDate",
+        parameters: [{ field: "category", op: "eq" }],
+      },
+      {
+        id: "posts-by-tag",
+        label: "Posts by tag",
+        datasourceId: "blog_posts",
+        limit: 10,
+        sortDirection: "desc",
+        sortField: "$publicationDate",
+        parameters: [{ field: "tags", op: "containsAny" }],
+      },
+      {
+        id: "author-posts",
+        label: "Author posts",
+        datasourceId: "blog_posts",
+        limit: 10,
+        sortDirection: "desc",
+        sortField: "$publicationDate",
+        parameters: [{ field: "author", op: "eq" }],
+      },
+      {
+        id: "search-posts",
+        label: "Search posts",
+        datasourceId: "blog_posts",
+        limit: 10,
+        sortDirection: "desc",
+        sortField: "$publicationDate",
+        parameters: [{ field: "title", op: "contains" }],
+      },
+    ],
+  },
+);
+
+export type Query = Static<typeof querySchema>;

package/src/shared/datasources/utils.ts

@@ -0,0 +1,17 @@
+import { get } from "lodash-es";
+import type { PageAttributes } from "../attributes";
+
+export function stringifyObjectValues(
+  obj: Record<string, string | number | Date | boolean>,
+): Record<string, string> {
+  return Object.fromEntries(Object.entries(obj).map(([key, value]) => [key, value.toString()]));
+}
+
+export function createPlaceholderReplacer(attr?: PageAttributes) {
+  return function replacePlaceholders(_: unknown, p1: string) {
+    const varName = (p1 as string).trim();
+    return String(get(attr, varName)) ?? "";
+  };
+}
+
+export const placeholderRx = /{{(.+?)}}/g;

package/src/shared/datasources.ts

@@ -0,0 +1,57 @@
+import type { Datasource } from "./datasources/types";
+import type { TArray } from "@sinclair/typebox";
+
+/**
+ * For now, defineDatasources() force the usage of a custom (internal) datasource
+ * @returns
+ */
+export function defineDatasource<D extends Datasource>(datasource: D) {
+  return {
+    ...datasource,
+    provider: "internal", // make sure to use Upstart provider
+    schema: mapDatasourceSchemaWithInternalProperties(
+      // datasource.provider === "internal" ? datasource.schema : getSchemaByProvider(datasource.provider),
+      datasource.schema,
+    ),
+  };
+}
+
+/**
+ * Map a datasource schema to include $id, $createdAt, and $updatedAt properties.
+ */
+export function mapDatasourceSchemaWithInternalProperties(schema: TArray): Datasource["schema"] {
+  const { items, ...rest } = schema;
+  return {
+    ...rest,
+    items: {
+      ...items,
+      type: "object",
+      properties: {
+        ...items.properties,
+        $id: { type: "string", title: "Id" },
+        $slug: { type: "string", format: "slug", title: "Slug" },
+        $publicationDate: { type: "string", format: "date-time", title: "Publication Date" },
+        $lastModificationDate: { type: "string", format: "date-time", title: "Last Modification Date" },
+      },
+      required: Array.from(
+        new Set(["$id", "$slug", "$publicationDate", "$lastModificationDate", ...items.required]),
+      ),
+    },
+  };
+}
+
+/**
+ * Extract the indexed fields of a datasource with their titles
+ */
+export function getDatasourceIndexedFieldsWithTitles(datasource: Datasource) {
+  if ("indexes" in datasource === false) {
+    console.error("no datasource or indexes found", datasource);
+    return [];
+  }
+  const properties = datasource.schema?.items?.properties || {};
+  const uniqueFields = Array.from(new Set(datasource.indexes?.map((idx) => idx.fields[0]) ?? []));
+  return uniqueFields.map((field) => ({
+    value: field,
+    title: properties[field]?.title || field,
+  }));
+}

package/src/shared/env.ts

@@ -0,0 +1,23 @@
+import type { D1Database, R2Bucket, KVNamespace } from "@cloudflare/workers-types";
+
+export type UpstartEnv = {
+  PUBLIC_UPSTART_SITE_ID: string;
+  PUBLIC_UPSTART_API_BASE_URL: string;
+  PUBLIC_UPSTART_SITE_HOST: string;
+  PUBLIC_UPSTART_ASSETS_BASE_URL?: string;
+  UPSTART_API_TOKEN: string;
+  SITES_DB: D1Database;
+  SITES_CACHE: KVNamespace;
+  R2_SITES_BUCKET: R2Bucket;
+  R2_SITES_BUCKET_NAME: string;
+  R2_ACCESS_KEY_ID: string;
+  R2_SECRET_ACCESS_KEY: string;
+  NODE_ENV: "development" | "production" | "preview" | "local-preview";
+  DATASOURCE_REFRESH_DELAY_IN_MINUTE: string;
+  FACEBOOK_APP_SECRET?: string;
+  INSTAGRAM_APP_SECRET?: string;
+  THREADS_APP_SECRET?: string;
+  TIKTOK_CLIENT_KEY?: string;
+  TIKTOK_CLIENT_SECRET?: string;
+  YOUTUBE_API_KEY?: string;
+};

package/src/shared/errors.ts

@@ -0,0 +1 @@
+export { UnauthorizedError } from "http-errors-enhanced";

package/src/shared/images.ts

@@ -0,0 +1,44 @@
+import { type Static, Type } from "@sinclair/typebox";
+import { StringEnum } from "./utils/string-enum";
+
+export const imageSearchSchema = Type.Object({
+  query: Type.String({
+    description: 'The search query for finding images (e.g. "mountain landscape", "coffee shop")',
+  }),
+  orientation: Type.Optional(
+    StringEnum(["landscape", "portrait", "squarish"], {
+      description: "Filter by photo orientation",
+    }),
+  ),
+});
+
+// Derive TypeScript type from the schema
+export type ImageSearchParams = Static<typeof imageSearchSchema>;
+
+export const imageResultsSchema = Type.Array(
+  Type.Object(
+    {
+      provider: Type.String({ description: "The image provider (e.g. unsplash, pexels)" }),
+      description: Type.String({ description: "A brief description of the image" }),
+      url: Type.String({ description: "The URL of the image" }),
+      blurHash: Type.String({ description: "The blur hash of the image" }),
+      user: Type.Object(
+        {
+          name: Type.String({ description: "The name of the user who uploaded the image" }),
+          profile_url: Type.String({ description: "The profile URL of the user who uploaded the image" }),
+        },
+        { additionalProperties: false },
+      ),
+    },
+    { additionalProperties: false },
+  ),
+  {
+    title: "Array of image search results",
+  },
+);
+
+export type ImageSearchResultsType = Static<typeof imageResultsSchema>;
+export type SimpleImageMetadata = Pick<
+  ImageSearchResultsType[number],
+  "description" | "blurHash" | "user" | "url" | "provider"
+>;

package/src/shared/index.ts

@@ -0,0 +1,3 @@
+export default {
+  name: "upstart-sdk",
+};

package/src/shared/layout-constants.ts

@@ -0,0 +1,19 @@
+export const LAYOUT_COLS = 24;
+
+export const LAYOUT_BREAKPOINTS = {
+  desktop: 1024,
+  mobile: 1,
+};
+
+/**
+ * Important: keep the same pdding for x and y.
+ */
+// export const LAYOUT_PADDING: Record<string, [number, number]> = {
+//   desktop: [20, 20],
+//   mobile: [8, 8],
+// };
+
+/**
+ * This is the height of a row in the layout grid.
+ */
+export const LAYOUT_ROW_HEIGHT = 24;