encore.dev 0.0.0-devel.202311141645

package/pubsub/subscription.ts

@@ -0,0 +1,302 @@
+import { api } from "@encore.dev/internal-runtime";
+import reqtrack, { Context } from "@encore.dev/internal-runtime/reqtrack/index";
+import {
+  newSpanID,
+  newTraceID,
+} from "@encore.dev/internal-runtime/reqtrack/tracecontext";
+import {
+  logCtx,
+  withLogCtx,
+} from "@encore.dev/internal-runtime/reqtrack/logging";
+import {
+  durationToProto,
+  now,
+  parseDuration,
+  since,
+  toDurationStr,
+} from "@encore.dev/internal-runtime/utils/timers";
+import { runtime } from "@encore.dev/internal-runtime/jsruntime";
+import log, { type Logger } from "../log/mod";
+import { PubSub } from "@encore.dev/sidecar-api";
+import { DurationString } from "../internal/types/mod";
+import { Acker } from "./acker";
+import { Topic } from "./topic";
+
+export class Subscription<Msg extends object> {
+  private readonly client_id: string;
+  private readonly topic: Topic<Msg>;
+  private readonly name: string;
+  private readonly handler: (msg: Msg, acker: Acker) => Promise<unknown>;
+  private readonly config: PubSub.SubscriptionConfig;
+  private readonly logger: Logger;
+  private failedState = false;
+  private activeDeliveries: string[] = [];
+
+  constructor(topic: Topic<Msg>, name: string, cfg: SubscriptionConfig<Msg>) {
+    // Generate a unique client ID for this instance
+    const id = new Uint8Array(16);
+    runtime().getRandomValues(id);
+    this.client_id = Buffer.from(id).toString("hex");
+
+    this.topic = topic;
+    this.name = name;
+    this.handler = cfg.handler;
+    this.logger = log.with({ topic: topic.name, subscription: name });
+
+    const ackDeadline = parseDuration(cfg.ackDeadline ?? "30s");
+    const minBackoff = parseDuration(cfg.retryPolicy?.minBackoff ?? "10s");
+    const maxBackoff = parseDuration(cfg.retryPolicy?.maxBackoff ?? "10m");
+
+    this.config = new PubSub.SubscriptionConfig({
+      maxConcurrency: cfg.maxConcurrency ?? -1,
+      ackDeadline: durationToProto(ackDeadline),
+      retryPolicy: new PubSub.RetryPolicy({
+        minBackoff: durationToProto(minBackoff),
+        maxBackoff: durationToProto(maxBackoff),
+        maxRetries: cfg.retryPolicy?.maxRetries ?? 100,
+      }),
+    });
+
+    let backoff = 250; // Start at 250ms for exponential backoff
+    const start = () => {
+      this.logger.trace("starting subscription");
+      this.#startSubscription()
+        .then(() => this.logger.warn("subscription ended"))
+        .catch((e: unknown) => {
+          this.failedState = true;
+          backoff *= 2;
+          if (backoff > 60 * 1000) {
+            // cap the backoff at 1 minute
+            backoff = 60 * 1000;
+          }
+
+          this.logger.error(e, "subscription failed with error");
+          setTimeout(start, backoff);
+        });
+    };
+
+    this.logger.info("registered subscription");
+
+    // Start the subscription
+    process.nextTick(start);
+  }
+
+  async #startSubscription() {
+    const resp = api().pubsub.subscribe(
+      new PubSub.SubscribeRequest({
+        clientId: this.client_id,
+        topic: this.topic.name,
+        subscription: this.name,
+        config: this.config,
+      }),
+    );
+
+    // Now we have an active subscription, we touch messages if we have active messages
+    const interval = setInterval(() => {
+      this.#touchActiveMessages();
+    }, 30 * 1000);
+
+    try {
+      for await (const msg of resp) {
+        if (this.failedState) {
+          this.logger.info("subscription recovered");
+          this.failedState = false;
+        }
+
+        process.nextTick(() => this.#handleMessage(msg));
+      }
+    } finally {
+      clearInterval(interval);
+    }
+  }
+
+  #touchActiveMessages() {
+    if (this.activeDeliveries.length === 0) {
+      return;
+    }
+    const acks = this.activeDeliveries;
+
+    api()
+      .pubsub.touch({
+        clientId: this.client_id,
+        ackIds: acks,
+      })
+      .then(() => {
+        this.logger.trace("touched active messages", { ack_ids: acks });
+      })
+      .catch((e: unknown) => {
+        this.logger.error(e, "failed to touch messages", { ack_ids: acks });
+      });
+  }
+
+  async #handleMessage(event: PubSub.SubscribeResponse) {
+    if (event.event.case !== "messageDelivery") {
+      // If it's a keep alive or some other unknown message
+      return;
+    }
+    const msg = event.event.value;
+
+    const logCtx: logCtx = {
+      topic: this.topic.name,
+      subscription: this.name,
+      msg_id: msg.msgId,
+      delivery_attempt: msg.deliveryAttempt,
+    };
+
+    const traceID = newTraceID();
+    const spanID = newSpanID();
+    const ctx: Context = {
+      traceID: traceID,
+      spanID: spanID,
+    };
+
+    // Record this ack ID as being active
+    this.activeDeliveries.push(msg.ackId);
+
+    return reqtrack.run(ctx, async () => {
+      return withLogCtx(logCtx, async () => {
+        const acker = new Acker(
+          () => this.#acknowledge(msg.ackId),
+          () => this.#negativeAcknowledge(msg.ackId),
+        );
+
+        try {
+          this.logger.info("starting request");
+          const data = Buffer.from(
+            msg.message?.data ?? Uint8Array.of(),
+          ).toString();
+
+          const start = now();
+          await this.handler(JSON.parse(data), acker);
+          this.logger.info("request completed", {
+            duration: toDurationStr(since(start)),
+          });
+
+          const hasAcked = await acker.currentResult();
+          if (!hasAcked) {
+            await acker.ack();
+          }
+        } catch (e: unknown) {
+          this.logger.error(e, "request failed");
+
+          const hasAcked = await acker.currentResult();
+          if (!hasAcked) {
+            await acker.nack();
+          }
+        }
+      });
+    });
+  }
+
+  async #acknowledge(ackID: string): Promise<void> {
+    await api().pubsub.acknowledge({
+      clientId: this.client_id,
+      ackId: ackID,
+    });
+    this.activeDeliveries = this.activeDeliveries.filter((id) => id !== ackID);
+    this.logger.trace("acknowledged message", { ack_id: ackID });
+  }
+
+  async #negativeAcknowledge(ackID: string): Promise<void> {
+    await api().pubsub.negativeAcknowledge({
+      clientId: this.client_id,
+      ackId: ackID,
+    });
+    this.activeDeliveries = this.activeDeliveries.filter((id) => id !== ackID);
+    this.logger.trace("negatively acknowledged message", { ack_id: ackID });
+  }
+}
+
+/**
+ * SubscriptionConfig is used when creating a subscription
+ *
+ * The values given here may be clamped to the supported values by
+ * the target cloud. (i.e. ack deadline may be brought within the supported range
+ * by the target cloud pubsub implementation).
+ */
+export interface SubscriptionConfig<Msg> {
+  /**
+   * Handler is the function which will be called to process a message
+   * sent on the topic.
+   *
+   * When this function returns an error the message will be
+   * negatively acknowledged (nacked), which will cause a redelivery
+   * attempt to be made (unless the retry policy's MaxRetries has been reached).
+   */
+  handler: (msg: Msg) => Promise<unknown>;
+
+  /**
+   * MaxConcurrency is the maximum number of messages which will be processed
+   * simultaneously per instance of the service for this subscription.
+   *
+   * Note that this is per instance of the service, so if your service has
+   * scaled to 10 instances and this is set to 10, then 100 messages could be
+   * processed simultaneously.
+   *
+   * If the value is negative, then there will be no limit on the number
+   * of messages processed simultaneously.
+   *
+   * Note: This is not supported by all cloud providers; specifically on GCP
+   * when using Cloud Run instances on an unordered topic the subscription will
+   * be configured as a Push Subscription and will have an adaptive concurrency
+   * See [GCP Push Delivery Rate](https://cloud.google.com/pubsub/docs/push#push_delivery_rate).
+   *
+   * This setting also has no effect on Encore Cloud environments.
+   * If not set, it uses a reasonable default based on the cloud provider.
+   */
+  maxConcurrency?: number;
+
+  /**
+   * AckDeadline is the time a consumer has to process a message
+   * before it's returned to the subscription
+   *
+   * Default is 30 seconds, however the ack deadline must be at least
+   * 1 second.
+   */
+  ackDeadline?: DurationString;
+
+  /**
+   * MessageRetention is how long an undelivered message is kept
+   * on the topic before it's purged.
+   *
+   * Default is 7 days.
+   */
+  messageRetention?: DurationString;
+
+  /**
+   * RetryPolicy defines how a message should be retried when
+   * the subscriber returns an error
+   */
+  retryPolicy?: RetryPolicy;
+}
+
+/**
+ * RetryPolicy defines how a subscription should handle retries
+ * after errors either delivering the message or processing the message.
+ *
+ * The values given to this structure are parsed at compile time, such that
+ * the correct Cloud resources can be provisioned to support the queue.
+ *
+ * As such the values given here may be clamped to the supported values by
+ * the target cloud. (i.e. min/max values brought within the supported range
+ * by the target cloud).
+ */
+export interface RetryPolicy {
+  /**
+   * The minimum time to wait between retries. Defaults to 10 seconds.
+   */
+  minBackoff?: DurationString;
+
+  /**
+   * The maximum time to wait between retries. Defaults to 10 minutes.
+   */
+  maxBackoff?: DurationString;
+
+  /**
+   * MaxRetries is used to control deadletter queuing logic, when:
+   *   n == 0: A default value of 100 retries will be used
+   *   n > 0:  Encore will forward a message to a dead letter queue after n retries
+   *   n == pubsub.InfiniteRetries: Messages will not be forwarded to the dead letter queue by the Encore framework
+   */
+  maxRetries?: number;
+}

package/pubsub/topic.ts

@@ -0,0 +1,132 @@
+import { api } from "@encore.dev/internal-runtime";
+import reqtrack from "@encore.dev/internal-runtime/reqtrack/index";
+import { PubSub } from "@encore.dev/sidecar-api";
+import { AttributesOf } from "./attributes";
+
+/**
+ * A topic is a resource to which you can publish messages
+ * to be delivered to subscribers of that topic.
+ */
+export class Topic<Msg extends object> {
+  public readonly name: string;
+  public readonly cfg: TopicConfig<Msg>;
+
+  constructor(name: string, cfg: TopicConfig<Msg>) {
+    this.name = name;
+    this.cfg = cfg;
+  }
+
+  public async publish(msg: Msg): Promise<string> {
+    const res = await api().pubsub.publish(
+      new PubSub.PublishRequest({
+        topic: this.name,
+        message: new PubSub.Message({
+          data: Buffer.from(JSON.stringify(msg)),
+          attributes: {},
+          traceParent: reqtrack.currentTraceParent(),
+        }),
+      }),
+    );
+
+    return res.id;
+  }
+}
+
+/**
+ * DeliveryGuarantee is used to configure the delivery contract for a topic.
+ */
+export type DeliveryGuarantee = "at-least-once" | "exactly-once";
+
+/**
+ * At Least Once delivery guarantees that a message for a subscription is delivered to
+ * a consumer at least once.
+ *
+ * On AWS and GCP there is no limit to the throughput for a topic.
+ */
+export const atLeastOnce: DeliveryGuarantee = "at-least-once";
+
+/**
+ * ExactlyOnce guarantees that a message for a subscription is delivered to
+ * a consumer exactly once, to the best of the system's ability.
+ *
+ * However, there are edge cases when a message might be redelivered.
+ * For example, if a networking issue causes the acknowledgement of success
+ * processing the message to be lost before the cloud provider receives it.
+ *
+ * It is also important to note that the ExactlyOnce delivery guarantee only
+ * applies to the delivery of the message to the consumer, and not to the
+ * original publishing of the message, such that if a message is published twice,
+ * such as due to an retry within the application logic, it will be delivered twice.
+ * (i.e. ExactlyOnce delivery does not imply message deduplication on publish)
+ *
+ * As such it's recommended that the subscription handler function is idempotent
+ * and is able to handle duplicate messages.
+ *
+ * Subscriptions attached to ExactlyOnce topics have higher message delivery latency compared to AtLeastOnce.
+ *
+ * By using ExactlyOnce semantics on a topic, the throughput will be limited depending on the cloud provider:
+ * - AWS: 300 messages per second for the topic (see [AWS SQS Quotas]).
+ * - GCP: At least 3,000 messages per second across all topics in the region
+ *      (can be higher on the region see [GCP PubSub Quotas]).
+ *
+ * [AWS SQS Quotas]: https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/quotas-messages.html
+ * [GCP PubSub Quotas]: https://cloud.google.com/pubsub/quotas#quotas
+ */
+export const exactlyOnce: DeliveryGuarantee = "exactly-once";
+
+/**
+ * TopicConfig is used when creating a Topic
+ */
+export interface TopicConfig<Msg extends object> {
+  /**
+   * DeliveryGuarantee is used to configure the delivery guarantee of a Topic
+   */
+  deliveryGuarantee: DeliveryGuarantee;
+
+  /**
+   * OrderingAttribute is the message attribute to use as a ordering key for
+   * messages and delivery will ensure that messages with the same value will
+   * be delivered in the order they where published.
+   *
+   * If OrderingAttribute is not set, messages can be delivered in any order.
+   *
+   * It is important to note, that in the case of an error being returned by a
+   * subscription handler, the message will be retried before any subsequent
+   * messages for that ordering key are delivered. This means depending on the
+   * retry configuration, a large backlog of messages for a given ordering key
+   * may build up. When using OrderingAttribute, it is recommended to use reason
+   * about your failure modes and set the retry configuration appropriately.
+   *
+   * Once the maximum number of retries has been reached, the message will be
+   * forwarded to the dead letter queue, and the next message for that ordering
+   * key will be delivered.
+   *
+   * To create attributes on a message, use the `Attribute` type:
+   *
+   *  type UserEvent = {
+   *    user_id Attribute<string>;
+   *    action  string;
+   *  }
+   *
+   *  const topic = new Topic<UserEvent>("user-events", {
+   *    deliveryGuarantee: DeliveryGuarantee.AtLeastOnce,
+   *    orderingAttribute: "user_id", // Messages with the same user-id will be delivered in the order they where
+   * published
+   *  })
+   *
+   *  topic.publish(ctx, {user_id: "1", action: "login"})  // This message will be delivered before the logout
+   *  topic.publish(ctx, {user_id: "2", action: "login"})  // This could be delivered at any time because it has a different user id
+   *  topic.publish(ctx, {user_id: "1", action: "logout"}) // This message will be delivered after the first message
+   *
+   * By using OrderingAttribute, the throughput will be limited depending on the cloud provider:
+   *
+   * - AWS: 300 messages per second for the topic (see [AWS SQS Quotas]).
+   * - GCP: 1MB/s for each ordering key (see [GCP PubSub Quotas]).
+   *
+   * Note: OrderingAttribute currently has no effect during local development.
+   *
+   * [AWS SQS Quotas]: https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/quotas-messages.html
+   * [GCP PubSub Quotas]: https://cloud.google.com/pubsub/quotas#resource_limits
+   */
+  orderingAttribute?: AttributesOf<Msg>;
+}

package/storage/sqldb/database.ts

@@ -0,0 +1,105 @@
+import { config } from "@encore.dev/internal-runtime";
+import type { StringLiteral } from "@encore.dev/internal-runtime/utils/constraints";
+import { queryWithComment } from "@encore.dev/internal-runtime/reqtrack/sql";
+import type { Meta } from "@encore.dev/sidecar-api";
+import pg from "pg";
+
+export interface SQLDatabaseConfig {
+  migrations?: string;
+}
+
+const driverName = "node-pg";
+
+/**
+ * Represents a single row from a query result
+ */
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export type Row = Record<string, any>;
+
+/** Represents a type that can be used in query template literals */
+export type Primitive = string | number | boolean | null;
+
+/**
+ * Constructing a new database object will result in Encore provisioning a database with
+ * that name and returning this object to represent it.
+ *
+ * If you want to reference an existing database, use `Database.Named(name)` as it is a
+ * compile error to create duplicate databases.
+ */
+export class SQLDatabase {
+  private readonly runtimeCfg: Meta.SimpleConfig_Infrastructure_SQLDatabaseConfig;
+  private readonly pool: pg.Pool;
+
+  /**
+   * Creates a new database with the given name and configuration
+   */
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  constructor(name: string, cfg?: SQLDatabaseConfig) {
+    // Grab the runtime configuration for this database
+    const runtimeCfg = config().infrastructure?.sqlDatabases[name];
+    if (!runtimeCfg) {
+      throw new Error(`No database configuration found for ${name}`);
+    }
+    this.runtimeCfg = runtimeCfg;
+
+    this.pool = new pg.Pool({
+      connectionString: runtimeCfg.connectionString,
+      max: 20,
+      idleTimeoutMillis: 30000,
+      connectionTimeoutMillis: 2000,
+    });
+  }
+
+  /**
+   * Reference an existing database by name, if the database doesn't
+   * exist yet, use `new Database(name)` instead.
+   */
+  static named<name extends string>(name: StringLiteral<name>): SQLDatabase {
+    return new SQLDatabase(name);
+  }
+
+  /**
+   * Returns the connection string for the database
+   */
+  get connectionString(): string {
+    return this.runtimeCfg.connectionString;
+  }
+
+  /**
+   * q allows you to query the database using a template string, replacing your placeholders in the template
+   * with parametrised values. It returns an async generator that you can iterate over to get the results, in a
+   * streaming fashion.
+   *
+   * @example
+   *
+   * const email = "foo@example.com";
+   * const result = database.q`SELECT id FROM users WHERE email=${email}` // produces a prepared query of
+   *                                                                      // "SELECT id FROM users WHERE email=$1"
+   *                                                                      // with an arg array of [email]
+   *
+   * for await (const row of result) {
+   *    console.log(row.id);
+   * }
+   */
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  async *q<T extends Row = Record<string, any>>(
+    strings: TemplateStringsArray,
+    ...expr: Primitive[]
+  ): AsyncGenerator<T> {
+    let query = "";
+    for (let i = 0; i < strings.length; i++) {
+      query += strings[i];
+
+      if (i < expr.length) {
+        query += "$" + (i + 1);
+      }
+    }
+
+    query = queryWithComment(query, driverName);
+
+    const result = await this.pool.query<T>(query, expr);
+    for (const row of result.rows) {
+      yield row;
+    }
+  }
+}

package/storage/sqldb/mod.ts

@@ -0,0 +1,2 @@
+export { SQLDatabase } from "./database";
+export type { SQLDatabaseConfig, Row as ResultRow } from "./database";